diff --git a/scripts/restore_large_files.py b/scripts/restore_large_files.py index 1c8422296..96a7a5010 100644 --- a/scripts/restore_large_files.py +++ b/scripts/restore_large_files.py @@ -40,6 +40,11 @@ def index_cache(cache_dir: str) -> dict[str, str]: """Map SHA1 to cached file path for every asset in the cache.""" index: dict[str, str] = {} for name in sorted(os.listdir(cache_dir)): + # A download in progress writes `..tmp` beside the + # cache and renames it when done: it is partial, and may be gone by + # the time it is opened. + if name.endswith(".tmp"): + continue path = os.path.join(cache_dir, name) if not os.path.isfile(path): continue diff --git a/tests/test_atomic_artifacts.py b/tests/test_atomic_artifacts.py index 5da433544..e106975b8 100644 --- a/tests/test_atomic_artifacts.py +++ b/tests/test_atomic_artifacts.py @@ -214,5 +214,34 @@ class ScratchFilesGetThePlainMode(unittest.TestCase): self.assertEqual(unset, []) + +class RestoreSkipsDownloadsInProgress(unittest.TestCase): + """A scratch file another run is still writing is neither hashed nor read.""" + + def test_a_scratch_promoted_mid_scan_does_not_stop_the_restore(self) -> None: + import restore_large_files + + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) + (cache / "done.bin").write_bytes(b"whole") + scratch = cache / "big.bin.ab12.tmp" + scratch.write_bytes(b"part") + real_isfile = os.path.isfile + + def check_then_promote(path: str) -> bool: + found = real_isfile(path) + # The concurrent fetch renames its scratch file right after + # the check, before the open. + if path == str(scratch) and found: + os.replace(scratch, cache / "big.bin") + return found + + with mock.patch.object(restore_large_files.os.path, "isfile", + side_effect=check_then_promote): + index = restore_large_files.index_cache(str(cache)) + self.assertNotIn(str(scratch), index.values()) + self.assertIn(str(cache / "done.bin"), index.values()) + + if __name__ == "__main__": unittest.main()