diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 88b06cf7..56d101a0 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -46,8 +46,10 @@ from common import ( artifact_lock, hold_artifact_lock, list_registered_platforms, + load_data_dir_registry, truth_target_dir, ) +from refresh_data_dirs import hold_data_dirs def run(cmd: list[str], label: str) -> tuple[bool, str]: @@ -478,6 +480,14 @@ def main() -> None: all_ok = _truth_steps(args, results) and all_ok + # Every step from here reads the data directory trees: verify, the packs, + # the manifests and the integrity check. Held shared until the run ends, + # a refresh from another session waits instead of swapping a tree + # between the packs and the manifests, which would then describe two + # revisions of it. Taken after step 2, whose own refresh needs them + # exclusive. + hold_data_dirs(held, load_data_dir_registry("platforms")) + # Step 3: Verify verify_cmd = [sys.executable, "scripts/verify.py", "--all", "--strict"] if args.include_archived: diff --git a/tests/test_artifact_lock.py b/tests/test_artifact_lock.py index abd4173c..748b109e 100644 --- a/tests/test_artifact_lock.py +++ b/tests/test_artifact_lock.py @@ -207,5 +207,19 @@ class HeldLockTest(unittest.TestCase): self.assertIn("hold_artifact_lock(args.output_dir)", source) +class PipelineHoldsTheDataTrees(unittest.TestCase): + """The pipeline held dist/ for the whole run but not the data directory + trees: a refresh from another session could swap one between the packs + and the manifests, which then described two revisions of it.""" + + def test_the_trees_are_held_after_the_refresh_and_before_verify(self): + source = (Path(__file__).resolve().parents[1] / "scripts" / "pipeline.py").read_text() + refresh = source.index('"scripts/refresh_data_dirs.py"') + hold = source.index("hold_data_dirs(held,") + verify = source.index('"scripts/verify.py", "--all"') + self.assertLess(refresh, hold) + self.assertLess(hold, verify) + + if __name__ == "__main__": unittest.main()