diff --git a/scripts/auto_fetch.py b/scripts/auto_fetch.py index 93434757..cb04c680 100644 --- a/scripts/auto_fetch.py +++ b/scripts/auto_fetch.py @@ -203,6 +203,18 @@ def step2_scan_branches(entry: dict) -> bytes | None: return None +# Sources that did not answer during the current file's search. A 404 is an +# answer, the file is not there. Anything else leaves the question open: an +# offline run used to report every file as needing a community contribution. +_UNANSWERED: list[str] = [] + + +def _unanswered(url: str, exc: Exception) -> None: + if isinstance(exc, urllib.error.HTTPError) and exc.code == 404: + return + _UNANSWERED.append(f"{url}: {exc}") + + def step3_search_public_repos(entry: dict) -> bytes | None: """Search public GitHub BIOS repos.""" name = entry["name"] @@ -220,7 +232,8 @@ def step3_search_public_repos(entry: dict) -> bytes | None: continue if verify_content(data, entry): return data - except (urllib.error.URLError, urllib.error.HTTPError): + except urllib.error.URLError as exc: + _unanswered(url, exc) continue if "/" in destination: @@ -235,7 +248,8 @@ def step3_search_public_repos(entry: dict) -> bytes | None: continue if verify_content(data, entry): return data - except (urllib.error.URLError, urllib.error.HTTPError): + except urllib.error.URLError as exc: + _unanswered(url, exc) continue return None @@ -258,7 +272,8 @@ def step4_search_archive_org(entry: dict) -> bytes | None: continue if verify_content(data, entry): return data - except (urllib.error.URLError, urllib.error.HTTPError): + except urllib.error.URLError as exc: + _unanswered(url, exc) continue sha1s = sorted(_accepted(entry, "sha1")) @@ -291,10 +306,10 @@ def step4_search_archive_org(entry: dict) -> bytes | None: data = _read_limited(resp2) if data is not None and verify_content(data, entry): return data - except (urllib.error.URLError, urllib.error.HTTPError): - pass - except (urllib.error.URLError, json.JSONDecodeError): - pass + except urllib.error.URLError as exc: + _unanswered(dl_url, exc) + except (urllib.error.URLError, json.JSONDecodeError) as exc: + _unanswered(search_url, exc) return None @@ -342,6 +357,7 @@ def fetch_missing( for entry in missing: name = entry["name"] + _UNANSWERED.clear() print(f"\n Searching: {name} ({entry['system']})") existing = step1_crossref_db(entry, db) @@ -377,6 +393,14 @@ def fetch_missing( stats["found"] += 1 continue + if _UNANSWERED: + # Not looked for everywhere: no issue asks the community for it. + print( + f" [5] Search incomplete, {len(_UNANSWERED)} source(s) did not " + f"answer (first: {_UNANSWERED[0]})" + ) + stats["errors"] += 1 + continue print(" [5] Not found - needs community contribution") still_missing.append(entry) stats["not_found"] += 1 @@ -485,7 +509,10 @@ def main() -> None: all_still_missing[platform] = result["still_missing"] stats = result["stats"] - print(f"\n Results: {stats['found']} found, {stats['not_found']} not found") + print( + f"\n Results: {stats['found']} found, {stats['not_found']} not found, " + f"{stats['errors']} not fully searched" + ) if args.create_issues and all_still_missing: print(f"\n{'=' * 60}") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 4418c94e..6163e97a 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5374,5 +5374,40 @@ class ABrokenRegistryIsNotAnEmptyOne(unittest.TestCase): self.assertIn("rocknix", str(caught.exception)) +class AnUnansweredSearchIsNotANotFound(unittest.TestCase): + """auto_fetch caught every network error and reported the file as + needing a community contribution: offline, each file became an issue + asking the public for something the tool never looked for.""" + + def _fetch(self, error): + from unittest import mock + + from scripts import auto_fetch + + def refuse(*_args, **_kwargs): + raise error + + entry = {"name": "nowhere.bin", "system": "test-system", "sha1": "0" * 40} + db = {"files": {}, "indexes": {"by_md5": {}, "by_name": {}}} + with mock.patch("urllib.request.urlopen", refuse), \ + mock.patch.object(auto_fetch, "step2_scan_branches", lambda e: None), \ + contextlib.redirect_stdout(io.StringIO()): + return auto_fetch.fetch_missing([entry], db, "bios") + + def test_offline_is_not_searched(self): + import urllib.error + + result = self._fetch(urllib.error.URLError("offline")) + self.assertEqual(result["still_missing"], []) + self.assertEqual(result["stats"]["errors"], 1) + + def test_a_404_everywhere_is_not_found(self): + import urllib.error + + result = self._fetch(urllib.error.HTTPError("u", 404, "missing", {}, None)) + self.assertEqual(len(result["still_missing"]), 1) + self.assertEqual(result["stats"]["errors"], 0) + + if __name__ == "__main__": unittest.main()