feat: keep a profile checkable when its forge goes

Four Switch profiles were unverifiable and said so on stderr every pass.
yuzu and suyu answer 451, citron's host stopped resolving, and
git.eden-emu.dev returns 403 to anything that is not a browser. Each
aborted its own report, so nothing could be said about any of them, and
the noise repeated on every run over the whole corpus.

A withdrawn forge is now a fact rather than a failure. GoneError covers
451, 410 and a host that does not resolve; none is retried, since three
attempts with backoff end in the same place. Those profiles land in
their own summary bucket, out of the review backlog where nobody could
act on them anyway. A 403 stays what it was, a refusal, because small
Forgejo instances behind anti-bot filters issue it routinely.

A profile can now name a source_mirror, consulted after source and
upstream so a live primary always decides attribution. Reaching it took
two more changes: a repository that refuses is muted for the rest of the
pass instead of ending it, keyed by host as well as slug because a
mirror carries the same slug on another forge; and a refused miss is not
cached, or the mute would answer for the mirror that was about to be
asked.

eden now reads from its Codeberg copy, which holds the same head and the
pinned commit: 5 refs, all anchored, where the profile could not be
checked at all. yuzu, suyu and citron have no mirror that serves
content, and now say so once instead of failing loudly.

scripts/profile_sync.py

@@ -1004,9 +1004,14 @@ def declared_repositories(profile: dict) -> list[tuple[str, str, str]]:
     `source` and `upstream` are usually plain strings. `ymir` keys them by
     build mode instead, because its standalone and libretro builds live in
     different repositories.
+
+    `source_mirror` comes last and is the only thing that keeps a profile
+    checkable once its forge is gone: yuzu and suyu answer 451, citron's
+    host stopped resolving. Being last, it never decides attribution while
+    a declared repository still answers.
     """
     urls: list[tuple[str, str, str]] = []
-    for field in ("source", "upstream"):
+    for field in ("source", "upstream", "source_mirror"):
         value = profile.get(field)
         if isinstance(value, dict):
             urls.extend((field, str(k), str(v)) for k, v in value.items() if v)
@@ -1274,7 +1279,7 @@ def build_report(
                     if matches:
                         break
         elif not any(
-            upstream.fetch_file(v.repo, sha, path, cache_dir, offline) is not None
+            fetch_from(v.repo, sha, path) is not None
             for v in views for sha in (v.pin, v.head)
         ):
             # A path written from a subproject directory of a monorepo:
@@ -1319,16 +1324,12 @@ def build_report(
         for candidate in candidates:
             for sha_of in (lambda v: v.pin, lambda v: v.head):
                 for view in views:
-                    found = upstream.fetch_file(
-                        view.repo, sha_of(view), candidate, cache_dir, offline
-                    )
+                    found = fetch_from(view.repo, sha_of(view), candidate)
                     if found is None:
                         continue
                     target = symlink_target(candidate, found)
                     if target:
-                        found = upstream.fetch_file(
-                            view.repo, sha_of(view), target, cache_dir, offline
-                        )
+                        found = fetch_from(view.repo, sha_of(view), target)
                         if found is None:
                             continue
                         candidate = target
@@ -1355,6 +1356,33 @@ def build_report(
             pin_trees[key] = tree
         return pin_trees[key]
 
+    mute: set[tuple[str, str]] = set()
+
+    def fetch_from(repo, sha, wanted):
+        """Read a file, treating a repository that refuses as one that lacks it.
+
+        A profile can name several repositories, and a mirror exists exactly
+        because one of them stopped answering: git.eden-emu.dev returns 403 to
+        anything that is not a browser. Letting the first refusal abort the
+        report leaves the mirror unread and the profile unverifiable, so the
+        repository is muted for the rest of the pass instead. A quota signal
+        still stops everything, because continuing would only burn the rest of
+        the budget on the same wall.
+        """
+        # Keyed by host as well as slug: a mirror carries the same slug on
+        # another forge, and muting one must not silence the other.
+        key = (repo.host, repo.slug)
+        if key in mute:
+            return None
+        try:
+            return upstream.fetch_file(repo, sha, wanted, cache_dir, offline)
+        except upstream.RateLimitError:
+            raise
+        except upstream.UpstreamError as exc:
+            mute.add(key)
+            print(f"{name}: {repo.host}/{repo.slug} muted, {exc}", file=sys.stderr)
+            return None
+
     def _context_for(view: RepoView):
         key = view.repo.slug
         if key not in context:
@@ -1409,9 +1437,14 @@ def build_report(
                 (forced, path) if forced else resolve_path(path, start, tokens)
             )
             sha = view.pin if which == PIN else view.head
-            lines_cache[key] = upstream.fetch_file(
-                view.repo, sha, actual, cache_dir, offline
-            )
+            found = fetch_from(view.repo, sha, actual)
+            if found is None:
+                # Not cached: the miss may be a repository that has just been
+                # muted, and the next call resolves the path to a mirror that
+                # does carry it. upstream.fetch_file holds its own cache, so
+                # asking again costs a lookup rather than a request.
+                return None
+            lines_cache[key] = found
         return lines_cache[key]
 
     def describe(path: str, start=None, tokens=(), forced=None):
@@ -1431,9 +1464,7 @@ def build_report(
         """
         best = None
         for view in views:
-            if upstream.fetch_file(
-                view.repo, view.pin, part.path, cache_dir, offline
-            ) is None:
+            if fetch_from(view.repo, view.pin, part.path) is None:
                 continue
             result = anchor_part(
                 part,
@@ -2780,6 +2811,14 @@ def main() -> None:
             report = build_report(name, profile, args.cache_dir, args.offline)
         except upstream.RateLimitError:
             raise
+        except upstream.GoneError as exc:
+            # The forge is not coming back: a takedown, or a host that no
+            # longer resolves. Saying so once in the summary beats repeating
+            # it on stderr every pass, and it keeps the profile out of the
+            # backlog, where nobody can act on it anyway.
+            report = ProfileReport(
+                name=name, entries=[], counts={}, skipped=f"upstream gone: {exc}"
+            )
         except upstream.UpstreamError as exc:
             # One unreachable forge must not abandon the other profiles.
             print(f"{name}: {exc}", file=sys.stderr)

scripts/upstream.py

@@ -11,6 +11,7 @@ import hashlib
 import http.client
 import json
 import os
+import socket
 import tempfile
 import time
 import urllib.error
@@ -69,6 +70,17 @@ class RateLimitError(UpstreamError):
     """The forge refused the request for quota reasons."""
 
 
+class GoneError(UpstreamError):
+    """The upstream is not coming back.
+
+    A legal takedown, a resource the forge reports as gone, or a host that
+    no longer resolves. Retrying costs time and ends in the same place, and
+    a caller sweeping every profile wants this told apart from a forge
+    having a bad minute: one is a fact about the project, the other is
+    weather.
+    """
+
+
 @dataclass(frozen=True)
 class Repo:
     host: str
@@ -161,6 +173,9 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError:
     """
     if exc.code == 429:
         return RateLimitError(f"{url}: HTTP 429")
+    if exc.code in (410, 451):
+        reason = "withdrawn for legal reasons" if exc.code == 451 else "gone"
+        return GoneError(f"{url}: HTTP {exc.code}, {reason}")
     if exc.code == 403 and exc.headers is not None:
         remaining = exc.headers.get("X-RateLimit-Remaining")
         if remaining is not None and remaining.strip() == "0":
@@ -168,6 +183,23 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError:
     return UpstreamError(f"{url}: HTTP {exc.code}")
 
 
+def _host_is_unresolvable(exc: BaseException) -> bool:
+    """Whether a connection failure is the name itself, not the network.
+
+    URLError carries the cause in `reason`, and a wrapped one carries it in
+    `__cause__`; the walk is bounded because either chain can be cyclic.
+    """
+    seen: BaseException | None = exc
+    for _ in range(8):
+        if seen is None:
+            break
+        if isinstance(seen, socket.gaierror):
+            return True
+        nested = getattr(seen, "reason", None)
+        seen = nested if isinstance(nested, BaseException) else seen.__cause__
+    return False
+
+
 def _fetch(url: str, accept_json: bool = False) -> bytes | None:
     """Body of a GET, or None on 404.
 
@@ -188,6 +220,8 @@ def _fetch(url: str, accept_json: bool = False) -> bytes | None:
             if isinstance(failure, RateLimitError) or exc.code < 500:
                 raise failure from exc
         except (urllib.error.URLError, http.client.HTTPException, OSError) as exc:
+            if _host_is_unresolvable(exc):
+                raise GoneError(f"{url}: host does not resolve") from exc
             failure = UpstreamError(f"{url}: {exc}")
         if attempt + 1 < RETRIES:
             _sleep(RETRY_BACKOFF[attempt])