diff --git a/scripts/pr_checks.py b/scripts/pr_checks.py index a5eba4f9e..518cb158d 100644 --- a/scripts/pr_checks.py +++ b/scripts/pr_checks.py @@ -33,6 +33,9 @@ class Step: # A second command that reads the first one's output, as a shell pipe # would: mypy's report goes through mypy-baseline filter. then: str = "" + # validate.yml marks the step continue-on-error: its failure is shown + # and does not fail the run. + may_fail: bool = False # The order of validate.yml: data fetched, site generated and built, then the @@ -48,7 +51,7 @@ STEPS: tuple[Step, ...] = ( Step("shellcheck --version", "the launcher test runs shellcheck", needs="shellcheck", provider="the shellcheck package"), Step("python scripts/export_native.py --all --fetch --output-dir tmp/native", - "files the exporter tests compare with"), + "files the exporter tests compare with", may_fail=True), Step("python scripts/refresh_data_dirs.py", "data directories the pack-building tests read"), Step("python scripts/generate_site.py", "the pages the site tests read"), Step("npm ci --prefix website", "the site's packages, Node 24", @@ -84,6 +87,7 @@ def execute(step: Step) -> int: def run(steps: tuple[Step, ...] = STEPS) -> int: failed: list[str] = [] + allowed: list[str] = [] not_run: list[str] = [] for step in steps: print(f"\n== {shown(step)}", flush=True) @@ -93,13 +97,15 @@ def run(steps: tuple[Step, ...] = STEPS) -> int: continue code = execute(step) if code: - failed.append(f"{shown(step)} (exit {code})") + (allowed if step.may_fail else failed).append(f"{shown(step)} (exit {code})") print() for line in failed: print(f"FAILED {line}") + for line in allowed: + print(f"FAILED {line}, continue-on-error in CI as well") for line in not_run: print(f"NOT RUN {line}") - print(f"{len(steps) - len(failed) - len(not_run)}/{len(steps)} checks passed") + print(f"{len(steps) - len(failed) - len(allowed) - len(not_run)}/{len(steps)} checks passed") return 1 if failed else 0 diff --git a/tests/test_contributing_text.py b/tests/test_contributing_text.py index 6c08e2f60..12431f02d 100644 --- a/tests/test_contributing_text.py +++ b/tests/test_contributing_text.py @@ -235,6 +235,16 @@ class OneCommandRunsTheChecks(unittest.TestCase): with self.subTest(check=check): self.assertTrue(any(command.startswith(check) for command in steps)) + def test_a_step_ci_lets_fail_does_not_fail_the_run(self): + import pr_checks + + workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + tolerated = set(re.findall(r"continue-on-error: true\n\s*run: (\S+ \S+)", workflow)) + self.assertTrue(tolerated) + for step in pr_checks.STEPS: + with self.subTest(step=step.command): + self.assertEqual(step.may_fail, " ".join(step.command.split()[:2]) in tolerated) + def test_steps_keep_ci_order(self): import pr_checks