feat: keep a profile checkable when its forge goes

Four Switch profiles were unverifiable and said so on stderr every pass.
yuzu and suyu answer 451, citron's host stopped resolving, and
git.eden-emu.dev returns 403 to anything that is not a browser. Each
aborted its own report, so nothing could be said about any of them, and
the noise repeated on every run over the whole corpus.

A withdrawn forge is now a fact rather than a failure. GoneError covers
451, 410 and a host that does not resolve; none is retried, since three
attempts with backoff end in the same place. Those profiles land in
their own summary bucket, out of the review backlog where nobody could
act on them anyway. A 403 stays what it was, a refusal, because small
Forgejo instances behind anti-bot filters issue it routinely.

A profile can now name a source_mirror, consulted after source and
upstream so a live primary always decides attribution. Reaching it took
two more changes: a repository that refuses is muted for the rest of the
pass instead of ending it, keyed by host as well as slug because a
mirror carries the same slug on another forge; and a refused miss is not
cached, or the mute would answer for the mirror that was about to be
asked.

eden now reads from its Codeberg copy, which holds the same head and the
pinned commit: 5 refs, all anchored, where the profile could not be
checked at all. yuzu, suyu and citron have no mirror that serves
content, and now say so once instead of failing loudly.

emulators/eden.yml

@@ -3,6 +3,9 @@ type: standalone
 core_classification: community_fork
 source: "https://git.eden-emu.dev/eden-emu/eden"
 upstream: "https://git.eden-emu.dev/eden-emu/eden"
+# git.eden-emu.dev answers 403 to anything that is not a browser, so the
+# Codeberg copy is what keeps this profile checkable.
+source_mirror: "https://codeberg.org/eden-emu/eden"
 profiled_date: "2026-03-26"
 source_commit: "c0ffc900cdf19b9373549c59a7e6b22c33615ea4"
 core_version: "0.1.1"

schemas/emulator.schema.json

@@ -597,6 +597,10 @@
     "source_branch": {
       "type": "string",
       "description": "Branch the port was read from, when it is not the default one."
+    },
+    "source_mirror": {
+      "type": "string",
+      "description": "Repository carrying the same tree, consulted after source and upstream. The only thing that keeps a profile checkable once its own forge is withdrawn or its host stops resolving."
     }
   }
 }

scripts/profile_sync.py

@@ -1004,9 +1004,14 @@ def declared_repositories(profile: dict) -> list[tuple[str, str, str]]:
     `source` and `upstream` are usually plain strings. `ymir` keys them by
     build mode instead, because its standalone and libretro builds live in
     different repositories.
+
+    `source_mirror` comes last and is the only thing that keeps a profile
+    checkable once its forge is gone: yuzu and suyu answer 451, citron's
+    host stopped resolving. Being last, it never decides attribution while
+    a declared repository still answers.
     """
     urls: list[tuple[str, str, str]] = []
-    for field in ("source", "upstream"):
+    for field in ("source", "upstream", "source_mirror"):
         value = profile.get(field)
         if isinstance(value, dict):
             urls.extend((field, str(k), str(v)) for k, v in value.items() if v)
@@ -1274,7 +1279,7 @@ def build_report(
                     if matches:
                         break
         elif not any(
-            upstream.fetch_file(v.repo, sha, path, cache_dir, offline) is not None
+            fetch_from(v.repo, sha, path) is not None
             for v in views for sha in (v.pin, v.head)
         ):
             # A path written from a subproject directory of a monorepo:
@@ -1319,16 +1324,12 @@ def build_report(
         for candidate in candidates:
             for sha_of in (lambda v: v.pin, lambda v: v.head):
                 for view in views:
-                    found = upstream.fetch_file(
-                        view.repo, sha_of(view), candidate, cache_dir, offline
-                    )
+                    found = fetch_from(view.repo, sha_of(view), candidate)
                     if found is None:
                         continue
                     target = symlink_target(candidate, found)
                     if target:
-                        found = upstream.fetch_file(
-                            view.repo, sha_of(view), target, cache_dir, offline
-                        )
+                        found = fetch_from(view.repo, sha_of(view), target)
                         if found is None:
                             continue
                         candidate = target
@@ -1355,6 +1356,33 @@ def build_report(
             pin_trees[key] = tree
         return pin_trees[key]
 
+    mute: set[tuple[str, str]] = set()
+
+    def fetch_from(repo, sha, wanted):
+        """Read a file, treating a repository that refuses as one that lacks it.
+
+        A profile can name several repositories, and a mirror exists exactly
+        because one of them stopped answering: git.eden-emu.dev returns 403 to
+        anything that is not a browser. Letting the first refusal abort the
+        report leaves the mirror unread and the profile unverifiable, so the
+        repository is muted for the rest of the pass instead. A quota signal
+        still stops everything, because continuing would only burn the rest of
+        the budget on the same wall.
+        """
+        # Keyed by host as well as slug: a mirror carries the same slug on
+        # another forge, and muting one must not silence the other.
+        key = (repo.host, repo.slug)
+        if key in mute:
+            return None
+        try:
+            return upstream.fetch_file(repo, sha, wanted, cache_dir, offline)
+        except upstream.RateLimitError:
+            raise
+        except upstream.UpstreamError as exc:
+            mute.add(key)
+            print(f"{name}: {repo.host}/{repo.slug} muted, {exc}", file=sys.stderr)
+            return None
+
     def _context_for(view: RepoView):
         key = view.repo.slug
         if key not in context:
@@ -1409,9 +1437,14 @@ def build_report(
                 (forced, path) if forced else resolve_path(path, start, tokens)
             )
             sha = view.pin if which == PIN else view.head
-            lines_cache[key] = upstream.fetch_file(
-                view.repo, sha, actual, cache_dir, offline
-            )
+            found = fetch_from(view.repo, sha, actual)
+            if found is None:
+                # Not cached: the miss may be a repository that has just been
+                # muted, and the next call resolves the path to a mirror that
+                # does carry it. upstream.fetch_file holds its own cache, so
+                # asking again costs a lookup rather than a request.
+                return None
+            lines_cache[key] = found
         return lines_cache[key]
 
     def describe(path: str, start=None, tokens=(), forced=None):
@@ -1431,9 +1464,7 @@ def build_report(
         """
         best = None
         for view in views:
-            if upstream.fetch_file(
-                view.repo, view.pin, part.path, cache_dir, offline
-            ) is None:
+            if fetch_from(view.repo, view.pin, part.path) is None:
                 continue
             result = anchor_part(
                 part,
@@ -2780,6 +2811,14 @@ def main() -> None:
             report = build_report(name, profile, args.cache_dir, args.offline)
         except upstream.RateLimitError:
             raise
+        except upstream.GoneError as exc:
+            # The forge is not coming back: a takedown, or a host that no
+            # longer resolves. Saying so once in the summary beats repeating
+            # it on stderr every pass, and it keeps the profile out of the
+            # backlog, where nobody can act on it anyway.
+            report = ProfileReport(
+                name=name, entries=[], counts={}, skipped=f"upstream gone: {exc}"
+            )
         except upstream.UpstreamError as exc:
             # One unreachable forge must not abandon the other profiles.
             print(f"{name}: {exc}", file=sys.stderr)

scripts/upstream.py

@@ -11,6 +11,7 @@ import hashlib
 import http.client
 import json
 import os
+import socket
 import tempfile
 import time
 import urllib.error
@@ -69,6 +70,17 @@ class RateLimitError(UpstreamError):
     """The forge refused the request for quota reasons."""
 
 
+class GoneError(UpstreamError):
+    """The upstream is not coming back.
+
+    A legal takedown, a resource the forge reports as gone, or a host that
+    no longer resolves. Retrying costs time and ends in the same place, and
+    a caller sweeping every profile wants this told apart from a forge
+    having a bad minute: one is a fact about the project, the other is
+    weather.
+    """
+
+
 @dataclass(frozen=True)
 class Repo:
     host: str
@@ -161,6 +173,9 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError:
     """
     if exc.code == 429:
         return RateLimitError(f"{url}: HTTP 429")
+    if exc.code in (410, 451):
+        reason = "withdrawn for legal reasons" if exc.code == 451 else "gone"
+        return GoneError(f"{url}: HTTP {exc.code}, {reason}")
     if exc.code == 403 and exc.headers is not None:
         remaining = exc.headers.get("X-RateLimit-Remaining")
         if remaining is not None and remaining.strip() == "0":
@@ -168,6 +183,23 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError:
     return UpstreamError(f"{url}: HTTP {exc.code}")
 
 
+def _host_is_unresolvable(exc: BaseException) -> bool:
+    """Whether a connection failure is the name itself, not the network.
+
+    URLError carries the cause in `reason`, and a wrapped one carries it in
+    `__cause__`; the walk is bounded because either chain can be cyclic.
+    """
+    seen: BaseException | None = exc
+    for _ in range(8):
+        if seen is None:
+            break
+        if isinstance(seen, socket.gaierror):
+            return True
+        nested = getattr(seen, "reason", None)
+        seen = nested if isinstance(nested, BaseException) else seen.__cause__
+    return False
+
+
 def _fetch(url: str, accept_json: bool = False) -> bytes | None:
     """Body of a GET, or None on 404.
 
@@ -188,6 +220,8 @@ def _fetch(url: str, accept_json: bool = False) -> bytes | None:
             if isinstance(failure, RateLimitError) or exc.code < 500:
                 raise failure from exc
         except (urllib.error.URLError, http.client.HTTPException, OSError) as exc:
+            if _host_is_unresolvable(exc):
+                raise GoneError(f"{url}: host does not resolve") from exc
             failure = UpstreamError(f"{url}: {exc}")
         if attempt + 1 < RETRIES:
             _sleep(RETRY_BACKOFF[attempt])

tests/test_profile_sync.py

@@ -218,6 +218,74 @@ class TestCollectTokens(unittest.TestCase):
         )
 
 
+class TestSourceMirror(unittest.TestCase):
+    """A profile whose forge is gone can name a mirror and stay checkable.
+
+    Nothing else recovers yuzu, suyu or citron: their trees exist, the host
+    that served them does not. The mirror is consulted last so a live
+    primary always decides attribution.
+    """
+
+    def test_mirror_is_declared_after_source_and_upstream(self):
+        profile = {
+            "source": "https://github.com/o/port",
+            "upstream": "https://github.com/o/up",
+            "source_mirror": "https://codeberg.org/o/mirror",
+        }
+        self.assertEqual(
+            [url for _, _, url in profile_sync.declared_repositories(profile)],
+            [
+                "https://github.com/o/port",
+                "https://github.com/o/up",
+                "https://codeberg.org/o/mirror",
+            ],
+        )
+
+    def test_mirror_alone_is_enough(self):
+        profile = {"source_mirror": "https://codeberg.org/o/mirror"}
+        self.assertEqual(
+            profile_sync.declared_repositories(profile),
+            [("source_mirror", "", "https://codeberg.org/o/mirror")],
+        )
+
+    def test_a_profile_without_a_mirror_is_unchanged(self):
+        profile = {"source": "https://github.com/o/port"}
+        self.assertEqual(
+            profile_sync.declared_repositories(profile),
+            [("source", "", "https://github.com/o/port")],
+        )
+
+
+class TestGoneUpstreamIsNotAnError(unittest.TestCase):
+    """A withdrawn forge is reported once, in its own bucket.
+
+    yuzu and suyu answer 451, citron's host no longer resolves. Printing
+    that on stderr every pass is noise nobody can act on, and counting it
+    with real upstream failures hides the forges that are merely having a
+    bad minute.
+    """
+
+    def test_a_gone_forge_is_skipped_not_errored(self):
+        report = ProfileReport(
+            name="yuzu", entries=[], counts={},
+            skipped="upstream gone: https://host/x: HTTP 451, withdrawn for legal reasons",
+        )
+        self.assertEqual(report.needs_review(), 0)
+        self.assertTrue(report.skipped.startswith("upstream gone:"))
+
+    def test_gone_and_failing_forges_land_in_different_buckets(self):
+        gone = ProfileReport(name="a", entries=[], counts={},
+                             skipped="upstream gone: h: HTTP 451, withdrawn for legal reasons")
+        failing = ProfileReport(name="b", entries=[], counts={},
+                                skipped="upstream error: h: HTTP 403")
+        buffer = io.StringIO()
+        with contextlib.redirect_stdout(buffer):
+            profile_sync._print_elided([gone, failing], 0)
+        out = buffer.getvalue()
+        self.assertIn("upstream gone", out)
+        self.assertIn("upstream error", out)
+
+
 class TestWorstStatus(unittest.TestCase):
     def test_gone_beats_everything(self):
         self.assertEqual(worst_status(["ANCHORED", "GONE", "SHIFTED"]), "GONE")

tests/test_upstream.py

@@ -7,6 +7,7 @@ import http.client
 import os
 import sys
 import tempfile
+import socket
 import unittest
 import urllib.error
 import urllib.request
@@ -245,6 +246,73 @@ class TestFetchRetry(unittest.TestCase):
         self.assertEqual(self.calls, 1)
 
 
+class TestGoneUpstream(unittest.TestCase):
+    """A forge that will not come back is a fact, not a failure to retry.
+
+    yuzu and suyu answer 451, citron's host no longer resolves. Retrying
+    those three times with backoff costs a minute per pass and still ends
+    in the same place, and calling them errors buries the profiles that
+    have something to say.
+    """
+
+    def setUp(self):
+        self._orig = (urllib.request.urlopen, upstream._sleep)
+        self.slept: list[float] = []
+        upstream._sleep = self.slept.append
+        self.calls = 0
+
+    def tearDown(self):
+        urllib.request.urlopen, upstream._sleep = self._orig
+
+    def _serve(self, outcome):
+        def opener(req, timeout=None):
+            self.calls += 1
+            raise outcome
+
+        urllib.request.urlopen = opener
+
+    def test_legal_takedown_is_gone_and_not_retried(self):
+        self._serve(_http_error(451))
+        with self.assertRaises(upstream.GoneError):
+            upstream._fetch("https://host/x")
+        self.assertEqual(self.calls, 1)
+        self.assertEqual(self.slept, [])
+
+    def test_http_gone_is_gone(self):
+        self._serve(_http_error(410))
+        with self.assertRaises(upstream.GoneError):
+            upstream._fetch("https://host/x")
+        self.assertEqual(self.calls, 1)
+
+    def test_unresolvable_host_is_gone_and_not_retried(self):
+        self._serve(
+            urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
+        )
+        with self.assertRaises(upstream.GoneError):
+            upstream._fetch("https://host/x")
+        self.assertEqual(self.calls, 1)
+        self.assertEqual(self.slept, [])
+
+    def test_a_gone_error_is_still_an_upstream_error(self):
+        self._serve(_http_error(451))
+        with self.assertRaises(upstream.UpstreamError):
+            upstream._fetch("https://host/x")
+
+    def test_refused_request_is_not_gone(self):
+        """403 is a forge refusing a request, which anti-bot filters do."""
+        self._serve(_http_error(403))
+        with self.assertRaises(upstream.UpstreamError) as caught:
+            upstream._fetch("https://host/x")
+        self.assertNotIsInstance(caught.exception, upstream.GoneError)
+
+    def test_a_dropped_connection_is_not_gone(self):
+        self._serve(http.client.RemoteDisconnected("closed"))
+        with self.assertRaises(upstream.UpstreamError) as caught:
+            upstream._fetch("https://host/x")
+        self.assertNotIsInstance(caught.exception, upstream.GoneError)
+        self.assertEqual(self.calls, upstream.RETRIES)
+
+
 class TestCache(unittest.TestCase):
     def setUp(self):
         self.tmp = tempfile.TemporaryDirectory()