diff --git a/scripts/pipeline.py b/scripts/pipeline.py index d59e1319..5972b72e 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -181,28 +181,18 @@ def check_consistency(verify_output: str, pack_output: str) -> bool: if v_total != p_total: print(f" {v_label}: MISMATCH total verify {v_total} != pack {p_total}") all_ok = False - elif p_ok + p_excluded < v_ok: - print( - f" {v_label}: MISMATCH pack accounts for " - f"{p_ok} OK + {p_excluded} unsafe exclusions " - f"< verify {v_ok} OK (/{v_total})" - ) - all_ok = False - elif p_ok < v_ok: - print( - f" {v_label}: verify {v_ok}/{v_total} native; pack {p_ok} safe, " - f"{p_excluded} unsafe excluded OK" - ) elif p_ok == v_ok: print( f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK" ) else: - # The invariant runs both ways: a builder more optimistic - # than the report ships files the report never vouched for. + # Both ways are a disagreement. A pack exclusion never covers + # a file verify counts OK: an existence platform ships a file + # whose hash disagrees, and a hash platform's verify reports + # it untested, so the counts can only differ by a defect. print( f" {v_label}: MISMATCH pack {p_ok}/{p_total} OK " - f"> verify {v_ok}/{v_total}" + f"({p_excluded} unsafe excluded) != verify {v_ok}/{v_total}" ) all_ok = False else: diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 7c4705f2..acc23989 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -457,7 +457,10 @@ class PipelineRegressions(unittest.TestCase): with contextlib.redirect_stdout(io.StringIO()): self.assertTrue(pipeline.check_consistency(verify, pack)) - def test_native_existence_and_strict_pack_exclusions_are_consistent(self): + def test_an_exclusion_never_covers_a_file_verify_counts_ok(self): + """An existence platform packs a file whose hash disagrees and a hash + platform's verify reports it untested: exclusions that make up the + difference hid the very divergence this step exists to catch.""" verify = "RetroArch: 3/3 present [existence]" pack = "\n".join( [ @@ -467,7 +470,7 @@ class PipelineRegressions(unittest.TestCase): ] ) with contextlib.redirect_stdout(io.StringIO()): - self.assertTrue(pipeline.check_consistency(verify, pack)) + self.assertFalse(pipeline.check_consistency(verify, pack)) self.assertEqual(pipeline.parse_pack_exclusions(pack), {"RetroArch": 1}) def test_unaccounted_pack_omission_is_a_consistency_failure(self):