diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 4e92d3941..795fbd62d 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -1154,11 +1154,19 @@ def select_views( branch = declared if pin_field == "source" else None head = None try: + unread: upstream.NotCachedError | None = None for candidate in branch_candidates(url): - head = upstream.resolve_head(repo, cache_dir, offline, candidate) + try: + head = upstream.resolve_head(repo, cache_dir, offline, candidate) + except upstream.NotCachedError as exc: + # Another prefix of the /tree/ tail may be the cached branch. + unread = exc + continue if head: branch = candidate break + if unread and not head: + raise unread pin, origin = resolve_pin( profile, repo, cache_dir, offline, pin_field, mode, branch ) diff --git a/scripts/upstream.py b/scripts/upstream.py index 131d0bc83..e514e980e 100644 --- a/scripts/upstream.py +++ b/scripts/upstream.py @@ -432,6 +432,22 @@ def fetch_file( MOVING_TTL = 6 * 3600 +def _api_target(url: str, cache_dir: str) -> Path: + """Where the cached answer to one API call lives.""" + key = hashlib.sha256(url.encode()).hexdigest() + return Path(cache_dir) / "_api" / f"{key}.json" + + +def _require_cached(url: str, cache_dir: str, offline: bool, what: str) -> None: + """Offline, refuse a call the cache holds no answer to, absence included. + + _api answers None both for a resource the forge does not have and for + one never asked, and only the first is a fact. + """ + if offline and not _api_target(url, cache_dir).is_file(): + raise NotCachedError(f"{what} is not in the offline cache") + + def _api( url: str, cache_dir: str, offline: bool, max_age: float | None = None, absent_max_age: float | None = None, @@ -443,8 +459,7 @@ def _api( ``absent_max_age`` bounds a cached absence alone, for a resource that can appear but, once there, does not change. """ - key = hashlib.sha256(url.encode()).hexdigest() - target = Path(cache_dir) / "_api" / f"{key}.json" + target = _api_target(url, cache_dir) if target.is_file(): raw = target.read_text(encoding="utf-8") bound = ( @@ -542,19 +557,31 @@ def resolve_head( default branch, so the code a profile documents can be absent from the tip the forge serves by default. ``fresh`` asks the forge again whatever the cache holds. + + Offline, a tip the cache never stored raises NotCachedError: returning + None read as a branch the forge does not have, and profile_sync dropped + xenios-jp/XeniOS, judging its citations against the other repository. """ - return _first_sha( - _api(_commits_url(repo, None, branch), cache_dir, offline, 0 if fresh else MOVING_TTL) + url = _commits_url(repo, None, branch) + _require_cached( + url, cache_dir, offline, f"{repo.slug}: tip of {branch or 'the default branch'}" ) + return _first_sha(_api(url, cache_dir, offline, 0 if fresh else MOVING_TTL)) def resolve_commit_at( repo: Repo, date: str, cache_dir: str, offline: bool = False, branch: str | None = None, ) -> str | None: - """Last commit on or before a date, on the default branch or a named one.""" + """Last commit on or before a date, on the default branch or a named one. + + Offline, an answer the cache never stored raises NotCachedError, as a + branch tip does: None means the forge has no such commit. + """ if repo.family in ("github", "gitlab"): - return _first_sha(_api(_commits_url(repo, date, branch), cache_dir, offline)) + url = _commits_url(repo, date, branch) + _require_cached(url, cache_dir, offline, f"{repo.slug}: commit at {date}") + return _first_sha(_api(url, cache_dir, offline)) return _gitea_commit_at(repo, date, cache_dir, offline, branch) @@ -581,7 +608,9 @@ def _gitea_commit_at( base += f"&sha={urllib.parse.quote(branch, safe='')}" page = 1 while True: - payload = _api(f"{base}&page={page}", cache_dir, offline, MOVING_TTL) + url = f"{base}&page={page}" + _require_cached(url, cache_dir, offline, f"{repo.slug}: commit at {date}") + payload = _api(url, cache_dir, offline, MOVING_TTL) if not isinstance(payload, list) or not payload: return None for entry in payload: diff --git a/tests/test_fileless_audit.py b/tests/test_fileless_audit.py index d8e179a93..7d469ab4f 100644 --- a/tests/test_fileless_audit.py +++ b/tests/test_fileless_audit.py @@ -184,9 +184,14 @@ class TestCorpusIsAnswered(unittest.TestCase): document = yaml_load(handle) or {} if document.get("files"): continue - if fileless_audit.audit( - path.stem, str(ROOT / "emulators"), str(ROOT / ".cache"), True - ): + try: + hits = fileless_audit.audit( + path.stem, str(ROOT / "emulators"), str(ROOT / ".cache"), True + ) + except fileless_audit.upstream.NotCachedError: + # A cold branch tip is unread too, not a branch gone. + continue + if hits: flagged.append(path.stem) if not read: self.skipTest("the upstream cache holds no source a fileless profile cites") diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index fc511761f..a9b32c370 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -296,6 +296,23 @@ class TestSourceMirror(unittest.TestCase): [("codeberg.org", "d16735f5")]) self.assertIn(("head", "codeberg.org", "dev"), calls) + def test_a_repository_the_cache_never_read_stops_the_profile(self): + """xenios offline with no cached tip of XeniOS's preview branch kept + has207/xenia-edge alone and judged XeniOS's files there: GONE, + "written against HEAD", 21 refs to review instead of 1.""" + import tempfile + + profile = { + "source": "https://github.com/xenios-jp/XeniOS", + "source_branch": "preview", + "upstream": "https://github.com/has207/xenia-edge", + "source_commit": "87b176a", + "upstream_commit": "07021f0", + } + with tempfile.TemporaryDirectory() as cache: + with self.assertRaises(profile_sync.upstream.NotCachedError): + profile_sync.select_views(profile, cache, True) + def test_a_profile_without_a_mirror_is_unchanged(self): profile = {"source": "https://github.com/o/port"} self.assertEqual( diff --git a/tests/test_upstream.py b/tests/test_upstream.py index b07d232d6..01817f52b 100644 --- a/tests/test_upstream.py +++ b/tests/test_upstream.py @@ -537,6 +537,21 @@ class TestRevisions(unittest.TestCase): ) self.assertIn("2026-03-29", self.calls[0]) + def test_an_uncached_tip_offline_is_unknown_not_absent(self): + """xenios-jp/XeniOS was dropped offline, its preview tip never + cached, and its citations were judged against the other repository.""" + with self.assertRaises(upstream.NotCachedError): + upstream.resolve_head(self.repo, self.dir, offline=True, branch="preview") + with self.assertRaises(upstream.NotCachedError): + upstream.resolve_commit_at(self.repo, "2026-03-29", self.dir, offline=True) + self.assertEqual(self.calls, []) + + def test_a_cached_absence_offline_is_still_absent(self): + self.assertIsNone(upstream.resolve_head(self.repo, self.dir, branch="gone")) + self.assertIsNone( + upstream.resolve_head(self.repo, self.dir, offline=True, branch="gone") + ) + def test_api_response_is_cached(self): self.responses["/commits"] = [{"sha": "abc"}] upstream.resolve_commit_at(self.repo, "2026-03-29", self.dir) @@ -588,10 +603,6 @@ class TestRevisions(unittest.TestCase): upstream.resolve_commit_at(self.repo, "2026-03-29", self.dir) self.assertEqual(len(self.calls), 1) - def test_offline_without_cache_returns_none(self): - self.assertIsNone(upstream.resolve_head(self.repo, self.dir, offline=True)) - self.assertEqual(self.calls, []) - def test_empty_history_returns_none(self): self.responses["/commits"] = [] self.assertIsNone(upstream.resolve_head(self.repo, self.dir))