diff --git a/scripts/pipeline.py b/scripts/pipeline.py index d068389e..d59e1319 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -198,9 +198,13 @@ def check_consistency(verify_output: str, pack_output: str) -> bool: f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK" ) else: + # The invariant runs both ways: a builder more optimistic + # than the report ships files the report never vouched for. print( - f" {v_label}: verify {v_ok}/{v_total}, pack {p_ok}/{p_total} OK (pack resolves more)" + f" {v_label}: MISMATCH pack {p_ok}/{p_total} OK " + f"> verify {v_ok}/{v_total}" ) + all_ok = False else: print(f" {v_label}: {v_ok}/{v_total} (no separate pack)") all_ok = False diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index ef3b2414..80d33955 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -482,6 +482,20 @@ class PipelineRegressions(unittest.TestCase): with contextlib.redirect_stdout(io.StringIO()): self.assertFalse(pipeline.check_consistency(verify, pack)) + def test_a_pack_more_optimistic_than_verify_is_a_failure(self): + """The invariant held one way only: a pack counting more OK files + than the report passed as 'pack resolves more'.""" + verify = "DupTest: 0/1 OK, 1 untested [md5]" + pack = "\n".join( + [ + "Generating pack for DupTest [source=full]...", + " x/DupTest_1_BIOS_Pack.zip: 1 files packed (1 baseline + 0 from cores), " + "1/1 files OK [md5]", + ] + ) + with contextlib.redirect_stdout(io.StringIO()): + self.assertFalse(pipeline.check_consistency(verify, pack)) + def test_nothing_parsed_is_not_consistent(self): """A changed output format must not pass with no platform compared.""" with contextlib.redirect_stdout(io.StringIO()):