fix: isolate an unreadable original per platform

A cached bios_db.json.zip that was not a zip raised BadZipFile out of
run(): --all stopped and the platforms after it were never exported.
fetch() also cached any 200 reply, so an error page replaced a good
archive and was served on every later run, and an IncompleteRead or an
oversized reply escaped the same way. An archive is now read before it
is cached, and an unreadable file fails its own platform with the
--refresh-cache remedy.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 15:24:30 +02:00
1 parent 194ed23ded
commit 4da40c471d
2 files changed
+106 -20

No files matched your search

+47 -20
View File
@@ -18,11 +18,15 @@ from __future__ import annotations
import argparse
import contextlib
import functools
import http.client
import re
import json
import sys
import urllib.error
import tarfile
import urllib.request
import zipfile
import zlib
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
@@ -82,8 +86,26 @@ def load_sources(upstream_dir: Path) -> dict[str, str]:
return _load_sources(upstream_dir / SOURCES_INDEX)
class OriginalUnreadable(ValueError):
"""A platform's file that is not what its format says it is."""
def _read_archive(unpack: Callable[[bytes], dict[str, str]], payload: bytes) -> dict[str, str]:
"""Unpack an original, naming the remedy when its bytes are not one."""
try:
return unpack(payload)
except (zipfile.BadZipFile, tarfile.TarError, EOFError, zlib.error, KeyError) as exc:
raise OriginalUnreadable(
f"archive unreadable ({exc}): --refresh-cache downloads it again"
) from exc
def fetch(
url: str, destination: Path, index: Path | None = None, refresh: bool = False
url: str,
destination: Path,
index: Path | None = None,
refresh: bool = False,
check: Callable[[bytes], object] | None = None,
) -> bytes:
"""Download an original once, then read it from the cache.
@@ -122,7 +144,11 @@ def fetch(
with urllib.request.urlopen(request, timeout=60) as response:
payload = response.read(_MAX_BYTES + 1)
if len(payload) > _MAX_BYTES:
raise ValueError(f"{url}: response larger than {_MAX_BYTES} bytes")
raise OriginalUnreadable(f"{url}: response larger than {_MAX_BYTES} bytes")
# An error page served with 200 replaced a good archive and was then
# served on every run: what does not read is never cached.
if check is not None:
check(payload)
destination.parent.mkdir(parents=True, exist_ok=True)
with locked():
write_bytes_atomic(str(destination), payload)
@@ -261,26 +287,27 @@ def collect_originals(
index = upstream_dir / SOURCES_INDEX
originals: dict[str, str] = {}
missing: list[str] = []
unpack = getattr(exporter, "unpack", None)
for relative, url in wanted.items():
path = root / relative
payload: bytes | None = None
if allow_fetch:
try:
payload = fetch(url, path, index, refresh)
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
missing.append(f"{relative}: {exc}")
archive = callable(unpack) and relative.endswith((".zip", ".tar.gz"))
check = (lambda raw: _read_archive(unpack, raw)) if archive else None
try:
if allow_fetch:
payload = fetch(url, path, index, refresh, check)
elif path.exists():
payload = path.read_bytes()
else:
missing.append(f"{relative}: absent and fetching is off")
continue
elif path.exists():
payload = path.read_bytes()
else:
missing.append(f"{relative}: absent and fetching is off")
continue
unpack = getattr(exporter, "unpack", None)
if callable(unpack) and relative.endswith((".zip", ".tar.gz")):
originals.update(unpack(payload))
else:
originals[relative] = payload.decode("utf-8", errors="replace")
if archive:
originals.update(_read_archive(unpack, payload))
else:
originals[relative] = payload.decode("utf-8", errors="replace")
# One platform's unreadable file fails that platform: raised, it
# stopped --all before the platforms after it.
except (OSError, http.client.HTTPException, OriginalUnreadable) as exc:
missing.append(f"{relative}: {exc}")
return originals, missing