diff --git a/scripts/upstream.py b/scripts/upstream.py index efcbfdff..f5970d9f 100644 --- a/scripts/upstream.py +++ b/scripts/upstream.py @@ -350,22 +350,27 @@ MOVING_TTL = 6 * 3600 def _api( - url: str, cache_dir: str, offline: bool, max_age: float | None = None + url: str, cache_dir: str, offline: bool, max_age: float | None = None, + absent_max_age: float | None = None, ) -> object | None: """Cached API call. The cache stores absence as well as payloads. ``max_age`` bounds the trust in a cached answer, in seconds; a stale answer is refreshed unless the run is offline, when it still serves. + ``absent_max_age`` bounds a cached absence alone, for a resource that + can appear but, once there, does not change. """ key = hashlib.sha256(url.encode()).hexdigest() target = Path(cache_dir) / "_api" / f"{key}.json" if target.is_file(): - fresh = ( - max_age is None - or time.time() - target.stat().st_mtime <= max_age + raw = target.read_text(encoding="utf-8") + bound = ( + absent_max_age + if raw == ABSENT and absent_max_age is not None + else max_age ) + fresh = bound is None or time.time() - target.stat().st_mtime <= bound if fresh or offline: - raw = target.read_text(encoding="utf-8") return None if raw == ABSENT else json.loads(raw) if offline: return None @@ -521,10 +526,13 @@ def tag_commit( if repo.family != "github": return resolve_tag_commit(repo, tag, cache_dir, offline) quoted = urllib.parse.quote(tag) + # A tag pushed after the first lookup must be seen: the 404 expires, + # the commit a tag names does not. payload = _api( f"{repo.api_base}/repos/{repo.slug}/git/ref/tags/{quoted}", cache_dir, offline, + absent_max_age=MOVING_TTL, ) if not isinstance(payload, dict): return None diff --git a/tests/test_upstream.py b/tests/test_upstream.py index fa08749d..dd0e5131 100644 --- a/tests/test_upstream.py +++ b/tests/test_upstream.py @@ -672,6 +672,30 @@ class TestCompare(unittest.TestCase): listing.extend([{"sha": "2"}, {"sha": "3"}]) self.assertEqual(upstream.commits_touching(self.repo, "a.c", "base", self.dir), 3) + def test_a_missing_tag_is_asked_again_once_stale(self): + """A profile written on release day asked for v2.0 before it was + pushed; the 404 was cached for good and the profile was never seen + as frozen on its tag.""" + import os as _os + + answers: list = [None] + + def answer(url): + return answers[-1] + + upstream._http_json = answer + self.assertIsNone(upstream.tag_commit(self.repo, "v2.0", self.dir)) + for cached in (Path(self.dir) / "_api").glob("*.json"): + stale = _os.path.getmtime(cached) - upstream.MOVING_TTL - 1 + _os.utime(cached, (stale, stale)) + answers.append({"object": {"type": "commit", "sha": "rel"}}) + self.assertEqual(upstream.tag_commit(self.repo, "v2.0", self.dir), "rel") + # Found, the answer is kept however old it gets. + for cached in (Path(self.dir) / "_api").glob("*.json"): + _os.utime(cached, (0, 0)) + answers.append({"object": {"type": "commit", "sha": "moved"}}) + self.assertEqual(upstream.tag_commit(self.repo, "v2.0", self.dir), "rel") + class TestGitlabTree(unittest.TestCase): def setUp(self):