diff --git a/emulators/eden.yml b/emulators/eden.yml index ba7829fc..dc7887f5 100644 --- a/emulators/eden.yml +++ b/emulators/eden.yml @@ -3,6 +3,9 @@ type: standalone core_classification: community_fork source: "https://git.eden-emu.dev/eden-emu/eden" upstream: "https://git.eden-emu.dev/eden-emu/eden" +# git.eden-emu.dev answers 403 to anything that is not a browser, so the +# Codeberg copy is what keeps this profile checkable. +source_mirror: "https://codeberg.org/eden-emu/eden" profiled_date: "2026-03-26" source_commit: "c0ffc900cdf19b9373549c59a7e6b22c33615ea4" core_version: "0.1.1" diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 7e2627de..ea965900 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -597,6 +597,10 @@ "source_branch": { "type": "string", "description": "Branch the port was read from, when it is not the default one." + }, + "source_mirror": { + "type": "string", + "description": "Repository carrying the same tree, consulted after source and upstream. The only thing that keeps a profile checkable once its own forge is withdrawn or its host stops resolving." } } } diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 402fab8e..421395c1 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -1004,9 +1004,14 @@ def declared_repositories(profile: dict) -> list[tuple[str, str, str]]: `source` and `upstream` are usually plain strings. `ymir` keys them by build mode instead, because its standalone and libretro builds live in different repositories. + + `source_mirror` comes last and is the only thing that keeps a profile + checkable once its forge is gone: yuzu and suyu answer 451, citron's + host stopped resolving. Being last, it never decides attribution while + a declared repository still answers. """ urls: list[tuple[str, str, str]] = [] - for field in ("source", "upstream"): + for field in ("source", "upstream", "source_mirror"): value = profile.get(field) if isinstance(value, dict): urls.extend((field, str(k), str(v)) for k, v in value.items() if v) @@ -1274,7 +1279,7 @@ def build_report( if matches: break elif not any( - upstream.fetch_file(v.repo, sha, path, cache_dir, offline) is not None + fetch_from(v.repo, sha, path) is not None for v in views for sha in (v.pin, v.head) ): # A path written from a subproject directory of a monorepo: @@ -1319,16 +1324,12 @@ def build_report( for candidate in candidates: for sha_of in (lambda v: v.pin, lambda v: v.head): for view in views: - found = upstream.fetch_file( - view.repo, sha_of(view), candidate, cache_dir, offline - ) + found = fetch_from(view.repo, sha_of(view), candidate) if found is None: continue target = symlink_target(candidate, found) if target: - found = upstream.fetch_file( - view.repo, sha_of(view), target, cache_dir, offline - ) + found = fetch_from(view.repo, sha_of(view), target) if found is None: continue candidate = target @@ -1355,6 +1356,33 @@ def build_report( pin_trees[key] = tree return pin_trees[key] + mute: set[tuple[str, str]] = set() + + def fetch_from(repo, sha, wanted): + """Read a file, treating a repository that refuses as one that lacks it. + + A profile can name several repositories, and a mirror exists exactly + because one of them stopped answering: git.eden-emu.dev returns 403 to + anything that is not a browser. Letting the first refusal abort the + report leaves the mirror unread and the profile unverifiable, so the + repository is muted for the rest of the pass instead. A quota signal + still stops everything, because continuing would only burn the rest of + the budget on the same wall. + """ + # Keyed by host as well as slug: a mirror carries the same slug on + # another forge, and muting one must not silence the other. + key = (repo.host, repo.slug) + if key in mute: + return None + try: + return upstream.fetch_file(repo, sha, wanted, cache_dir, offline) + except upstream.RateLimitError: + raise + except upstream.UpstreamError as exc: + mute.add(key) + print(f"{name}: {repo.host}/{repo.slug} muted, {exc}", file=sys.stderr) + return None + def _context_for(view: RepoView): key = view.repo.slug if key not in context: @@ -1409,9 +1437,14 @@ def build_report( (forced, path) if forced else resolve_path(path, start, tokens) ) sha = view.pin if which == PIN else view.head - lines_cache[key] = upstream.fetch_file( - view.repo, sha, actual, cache_dir, offline - ) + found = fetch_from(view.repo, sha, actual) + if found is None: + # Not cached: the miss may be a repository that has just been + # muted, and the next call resolves the path to a mirror that + # does carry it. upstream.fetch_file holds its own cache, so + # asking again costs a lookup rather than a request. + return None + lines_cache[key] = found return lines_cache[key] def describe(path: str, start=None, tokens=(), forced=None): @@ -1431,9 +1464,7 @@ def build_report( """ best = None for view in views: - if upstream.fetch_file( - view.repo, view.pin, part.path, cache_dir, offline - ) is None: + if fetch_from(view.repo, view.pin, part.path) is None: continue result = anchor_part( part, @@ -2780,6 +2811,14 @@ def main() -> None: report = build_report(name, profile, args.cache_dir, args.offline) except upstream.RateLimitError: raise + except upstream.GoneError as exc: + # The forge is not coming back: a takedown, or a host that no + # longer resolves. Saying so once in the summary beats repeating + # it on stderr every pass, and it keeps the profile out of the + # backlog, where nobody can act on it anyway. + report = ProfileReport( + name=name, entries=[], counts={}, skipped=f"upstream gone: {exc}" + ) except upstream.UpstreamError as exc: # One unreachable forge must not abandon the other profiles. print(f"{name}: {exc}", file=sys.stderr) diff --git a/scripts/upstream.py b/scripts/upstream.py index ae294aed..9e6e900d 100644 --- a/scripts/upstream.py +++ b/scripts/upstream.py @@ -11,6 +11,7 @@ import hashlib import http.client import json import os +import socket import tempfile import time import urllib.error @@ -69,6 +70,17 @@ class RateLimitError(UpstreamError): """The forge refused the request for quota reasons.""" +class GoneError(UpstreamError): + """The upstream is not coming back. + + A legal takedown, a resource the forge reports as gone, or a host that + no longer resolves. Retrying costs time and ends in the same place, and + a caller sweeping every profile wants this told apart from a forge + having a bad minute: one is a fact about the project, the other is + weather. + """ + + @dataclass(frozen=True) class Repo: host: str @@ -161,6 +173,9 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError: """ if exc.code == 429: return RateLimitError(f"{url}: HTTP 429") + if exc.code in (410, 451): + reason = "withdrawn for legal reasons" if exc.code == 451 else "gone" + return GoneError(f"{url}: HTTP {exc.code}, {reason}") if exc.code == 403 and exc.headers is not None: remaining = exc.headers.get("X-RateLimit-Remaining") if remaining is not None and remaining.strip() == "0": @@ -168,6 +183,23 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError: return UpstreamError(f"{url}: HTTP {exc.code}") +def _host_is_unresolvable(exc: BaseException) -> bool: + """Whether a connection failure is the name itself, not the network. + + URLError carries the cause in `reason`, and a wrapped one carries it in + `__cause__`; the walk is bounded because either chain can be cyclic. + """ + seen: BaseException | None = exc + for _ in range(8): + if seen is None: + break + if isinstance(seen, socket.gaierror): + return True + nested = getattr(seen, "reason", None) + seen = nested if isinstance(nested, BaseException) else seen.__cause__ + return False + + def _fetch(url: str, accept_json: bool = False) -> bytes | None: """Body of a GET, or None on 404. @@ -188,6 +220,8 @@ def _fetch(url: str, accept_json: bool = False) -> bytes | None: if isinstance(failure, RateLimitError) or exc.code < 500: raise failure from exc except (urllib.error.URLError, http.client.HTTPException, OSError) as exc: + if _host_is_unresolvable(exc): + raise GoneError(f"{url}: host does not resolve") from exc failure = UpstreamError(f"{url}: {exc}") if attempt + 1 < RETRIES: _sleep(RETRY_BACKOFF[attempt]) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 58dd5405..9af5227b 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -218,6 +218,74 @@ class TestCollectTokens(unittest.TestCase): ) +class TestSourceMirror(unittest.TestCase): + """A profile whose forge is gone can name a mirror and stay checkable. + + Nothing else recovers yuzu, suyu or citron: their trees exist, the host + that served them does not. The mirror is consulted last so a live + primary always decides attribution. + """ + + def test_mirror_is_declared_after_source_and_upstream(self): + profile = { + "source": "https://github.com/o/port", + "upstream": "https://github.com/o/up", + "source_mirror": "https://codeberg.org/o/mirror", + } + self.assertEqual( + [url for _, _, url in profile_sync.declared_repositories(profile)], + [ + "https://github.com/o/port", + "https://github.com/o/up", + "https://codeberg.org/o/mirror", + ], + ) + + def test_mirror_alone_is_enough(self): + profile = {"source_mirror": "https://codeberg.org/o/mirror"} + self.assertEqual( + profile_sync.declared_repositories(profile), + [("source_mirror", "", "https://codeberg.org/o/mirror")], + ) + + def test_a_profile_without_a_mirror_is_unchanged(self): + profile = {"source": "https://github.com/o/port"} + self.assertEqual( + profile_sync.declared_repositories(profile), + [("source", "", "https://github.com/o/port")], + ) + + +class TestGoneUpstreamIsNotAnError(unittest.TestCase): + """A withdrawn forge is reported once, in its own bucket. + + yuzu and suyu answer 451, citron's host no longer resolves. Printing + that on stderr every pass is noise nobody can act on, and counting it + with real upstream failures hides the forges that are merely having a + bad minute. + """ + + def test_a_gone_forge_is_skipped_not_errored(self): + report = ProfileReport( + name="yuzu", entries=[], counts={}, + skipped="upstream gone: https://host/x: HTTP 451, withdrawn for legal reasons", + ) + self.assertEqual(report.needs_review(), 0) + self.assertTrue(report.skipped.startswith("upstream gone:")) + + def test_gone_and_failing_forges_land_in_different_buckets(self): + gone = ProfileReport(name="a", entries=[], counts={}, + skipped="upstream gone: h: HTTP 451, withdrawn for legal reasons") + failing = ProfileReport(name="b", entries=[], counts={}, + skipped="upstream error: h: HTTP 403") + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + profile_sync._print_elided([gone, failing], 0) + out = buffer.getvalue() + self.assertIn("upstream gone", out) + self.assertIn("upstream error", out) + + class TestWorstStatus(unittest.TestCase): def test_gone_beats_everything(self): self.assertEqual(worst_status(["ANCHORED", "GONE", "SHIFTED"]), "GONE") diff --git a/tests/test_upstream.py b/tests/test_upstream.py index 51ab6133..828fed10 100644 --- a/tests/test_upstream.py +++ b/tests/test_upstream.py @@ -7,6 +7,7 @@ import http.client import os import sys import tempfile +import socket import unittest import urllib.error import urllib.request @@ -245,6 +246,73 @@ class TestFetchRetry(unittest.TestCase): self.assertEqual(self.calls, 1) +class TestGoneUpstream(unittest.TestCase): + """A forge that will not come back is a fact, not a failure to retry. + + yuzu and suyu answer 451, citron's host no longer resolves. Retrying + those three times with backoff costs a minute per pass and still ends + in the same place, and calling them errors buries the profiles that + have something to say. + """ + + def setUp(self): + self._orig = (urllib.request.urlopen, upstream._sleep) + self.slept: list[float] = [] + upstream._sleep = self.slept.append + self.calls = 0 + + def tearDown(self): + urllib.request.urlopen, upstream._sleep = self._orig + + def _serve(self, outcome): + def opener(req, timeout=None): + self.calls += 1 + raise outcome + + urllib.request.urlopen = opener + + def test_legal_takedown_is_gone_and_not_retried(self): + self._serve(_http_error(451)) + with self.assertRaises(upstream.GoneError): + upstream._fetch("https://host/x") + self.assertEqual(self.calls, 1) + self.assertEqual(self.slept, []) + + def test_http_gone_is_gone(self): + self._serve(_http_error(410)) + with self.assertRaises(upstream.GoneError): + upstream._fetch("https://host/x") + self.assertEqual(self.calls, 1) + + def test_unresolvable_host_is_gone_and_not_retried(self): + self._serve( + urllib.error.URLError(socket.gaierror(-2, "Name or service not known")) + ) + with self.assertRaises(upstream.GoneError): + upstream._fetch("https://host/x") + self.assertEqual(self.calls, 1) + self.assertEqual(self.slept, []) + + def test_a_gone_error_is_still_an_upstream_error(self): + self._serve(_http_error(451)) + with self.assertRaises(upstream.UpstreamError): + upstream._fetch("https://host/x") + + def test_refused_request_is_not_gone(self): + """403 is a forge refusing a request, which anti-bot filters do.""" + self._serve(_http_error(403)) + with self.assertRaises(upstream.UpstreamError) as caught: + upstream._fetch("https://host/x") + self.assertNotIsInstance(caught.exception, upstream.GoneError) + + def test_a_dropped_connection_is_not_gone(self): + self._serve(http.client.RemoteDisconnected("closed")) + with self.assertRaises(upstream.UpstreamError) as caught: + upstream._fetch("https://host/x") + self.assertNotIsInstance(caught.exception, upstream.GoneError) + self.assertEqual(self.calls, upstream.RETRIES) + + class TestCache(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory()