diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index f82966b5..35486f37 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -959,6 +959,15 @@ def main() -> None: if args.platform: require_platform(args.platform, args.platforms_dir) + import contextlib + + from refresh_data_dirs import hold_data_dirs + + # Files resolve against data/: the trees stay put until the process + # ends, which releases the hold. + held = contextlib.ExitStack() + hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) + profiles = load_emulator_profiles(args.emulators_dir) emulator = args.emulator if emulator: diff --git a/scripts/export_native.py b/scripts/export_native.py index 1ccddb68..47db1e0a 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -898,6 +898,17 @@ def main() -> None: require_platform(args.platform, args.platforms_dir) platforms = [args.platform] + if not args.refresh_cache: + import contextlib + + from refresh_data_dirs import hold_data_dirs + + # Removal candidates are proved against data/: a refresh swapping a + # tree mid-walk listed entries only that tree holds as removable. + # The process exit releases the hold. + held = contextlib.ExitStack() + hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) + sys.exit( run( platforms, diff --git a/scripts/refresh_data_dirs.py b/scripts/refresh_data_dirs.py index f1f0c4b5..53e40891 100644 --- a/scripts/refresh_data_dirs.py +++ b/scripts/refresh_data_dirs.py @@ -99,11 +99,13 @@ def hold_data_dirs(stack: contextlib.ExitStack, registry: dict) -> None: later. A refresh swapping the tree in between shipped bytes nobody verified, or died on a file the new tree no longer has. A refresh waits for the holders. Never called before a refresh in the same process: - the exclusive lock would wait on this one. + the exclusive lock would wait on this one. A cache not yet on disk is + held too: the lock sits beside the tree, and skipped, a refresh could + install the tree after verify ran and before the packs read it. """ for entry in registry.values(): cache = entry.get("local_cache", "") - if cache and os.path.isdir(cache): + if cache: stack.enter_context(cache_lock(cache, shared=True)) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index d12efd68..f5afb746 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3761,6 +3761,21 @@ class ADataDirectoryIsHeldWhileRead(unittest.TestCase): self.assertFalse(exclusive_free()) self.assertTrue(exclusive_free()) + def test_a_cache_not_yet_on_disk_is_held_too(self): + """Skipped while absent, a tree a refresh installed mid-run was + missing for verify and present for the packs of the same run.""" + import fcntl + + from scripts import refresh_data_dirs + + cache = self.tmp / "absent" + lock = cache.with_name(f".{cache.name}.lock") + with contextlib.ExitStack() as held: + refresh_data_dirs.hold_data_dirs(held, {"absent": {"local_cache": str(cache)}}) + self.assertFalse(cache.exists()) + with open(lock, "a") as handle, self.assertRaises(BlockingIOError): + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + def test_both_entry_points_hold_the_trees(self): pack = (ROOT / "scripts" / "generate_pack.py").read_text() verify_src = (ROOT / "scripts" / "verify.py").read_text() @@ -5446,7 +5461,8 @@ class ReportsHoldTheDataTrees(unittest.TestCase): def test_each_entry_point_holds_them(self): import ast - for script in ("generate_readme.py", "generate_site.py", "slots.py"): + for script in ("generate_readme.py", "generate_site.py", "slots.py", + "export_native.py", "cross_reference.py"): tree = ast.parse((ROOT / "scripts" / script).read_text(encoding="utf-8")) main = next( node for node in tree.body