fix: an unanswered search is not a missing file

auto_fetch caught every network error in its repository and archive.org steps and reported the file as needing a community contribution. Offline or rate-limited, --create-issues asked the public for files the tool never looked for, and the errors counter stayed at 0. A 404 is still an answer. Any other failure now marks the file as not fully searched and keeps it out of issues.

scripts/auto_fetch.py

@@ -203,6 +203,18 @@ def step2_scan_branches(entry: dict) -> bytes | None:
     return None
 
 
+# Sources that did not answer during the current file's search. A 404 is an
+# answer, the file is not there. Anything else leaves the question open: an
+# offline run used to report every file as needing a community contribution.
+_UNANSWERED: list[str] = []
+
+
+def _unanswered(url: str, exc: Exception) -> None:
+    if isinstance(exc, urllib.error.HTTPError) and exc.code == 404:
+        return
+    _UNANSWERED.append(f"{url}: {exc}")
+
+
 def step3_search_public_repos(entry: dict) -> bytes | None:
     """Search public GitHub BIOS repos."""
     name = entry["name"]
@@ -220,7 +232,8 @@ def step3_search_public_repos(entry: dict) -> bytes | None:
                     continue
                 if verify_content(data, entry):
                     return data
-        except (urllib.error.URLError, urllib.error.HTTPError):
+        except urllib.error.URLError as exc:
+            _unanswered(url, exc)
             continue
 
         if "/" in destination:
@@ -235,7 +248,8 @@ def step3_search_public_repos(entry: dict) -> bytes | None:
                         continue
                     if verify_content(data, entry):
                         return data
-            except (urllib.error.URLError, urllib.error.HTTPError):
+            except urllib.error.URLError as exc:
+                _unanswered(url, exc)
                 continue
 
     return None
@@ -258,7 +272,8 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
                         continue
                     if verify_content(data, entry):
                         return data
-            except (urllib.error.URLError, urllib.error.HTTPError):
+            except urllib.error.URLError as exc:
+                _unanswered(url, exc)
                 continue
 
     sha1s = sorted(_accepted(entry, "sha1"))
@@ -291,10 +306,10 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
                             data = _read_limited(resp2)
                             if data is not None and verify_content(data, entry):
                                 return data
-                    except (urllib.error.URLError, urllib.error.HTTPError):
-                        pass
-    except (urllib.error.URLError, json.JSONDecodeError):
-        pass
+                    except urllib.error.URLError as exc:
+                        _unanswered(dl_url, exc)
+    except (urllib.error.URLError, json.JSONDecodeError) as exc:
+        _unanswered(search_url, exc)
 
     return None
 
@@ -342,6 +357,7 @@ def fetch_missing(
 
     for entry in missing:
         name = entry["name"]
+        _UNANSWERED.clear()
         print(f"\n  Searching: {name} ({entry['system']})")
 
         existing = step1_crossref_db(entry, db)
@@ -377,6 +393,14 @@ def fetch_missing(
             stats["found"] += 1
             continue
 
+        if _UNANSWERED:
+            # Not looked for everywhere: no issue asks the community for it.
+            print(
+                f"    [5] Search incomplete, {len(_UNANSWERED)} source(s) did not "
+                f"answer (first: {_UNANSWERED[0]})"
+            )
+            stats["errors"] += 1
+            continue
         print("    [5] Not found - needs community contribution")
         still_missing.append(entry)
         stats["not_found"] += 1
@@ -485,7 +509,10 @@ def main() -> None:
             all_still_missing[platform] = result["still_missing"]
 
         stats = result["stats"]
-        print(f"\n  Results: {stats['found']} found, {stats['not_found']} not found")
+        print(
+            f"\n  Results: {stats['found']} found, {stats['not_found']} not found, "
+            f"{stats['errors']} not fully searched"
+        )
 
     if args.create_issues and all_still_missing:
         print(f"\n{'=' * 60}")

tests/test_audit_regressions.py

@@ -5374,5 +5374,40 @@ class ABrokenRegistryIsNotAnEmptyOne(unittest.TestCase):
             self.assertIn("rocknix", str(caught.exception))
 
 
+class AnUnansweredSearchIsNotANotFound(unittest.TestCase):
+    """auto_fetch caught every network error and reported the file as
+    needing a community contribution: offline, each file became an issue
+    asking the public for something the tool never looked for."""
+
+    def _fetch(self, error):
+        from unittest import mock
+
+        from scripts import auto_fetch
+
+        def refuse(*_args, **_kwargs):
+            raise error
+
+        entry = {"name": "nowhere.bin", "system": "test-system", "sha1": "0" * 40}
+        db = {"files": {}, "indexes": {"by_md5": {}, "by_name": {}}}
+        with mock.patch("urllib.request.urlopen", refuse), \
+                mock.patch.object(auto_fetch, "step2_scan_branches", lambda e: None), \
+                contextlib.redirect_stdout(io.StringIO()):
+            return auto_fetch.fetch_missing([entry], db, "bios")
+
+    def test_offline_is_not_searched(self):
+        import urllib.error
+
+        result = self._fetch(urllib.error.URLError("offline"))
+        self.assertEqual(result["still_missing"], [])
+        self.assertEqual(result["stats"]["errors"], 1)
+
+    def test_a_404_everywhere_is_not_found(self):
+        import urllib.error
+
+        result = self._fetch(urllib.error.HTTPError("u", 404, "missing", {}, None))
+        self.assertEqual(len(result["still_missing"]), 1)
+        self.assertEqual(result["stats"]["errors"], 0)
+
+
 if __name__ == "__main__":
     unittest.main()