diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 02c53735b..ab7dfdea1 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -603,6 +603,26 @@ def _refuse_unapplied_flags( args.source = "full" +def _refuse_offline_without_caches( + parser: argparse.ArgumentParser, args: argparse.Namespace, platforms_dir: str = "platforms", +) -> None: + """An offline run that builds packs needs the data directories cached. + + On a fresh clone none is: --offline skips the refresh, and the run went + through the database, verify and every pack before failing at the end + of the build. Only an empty cache is refused here: with some cached, the + packs decide which ones they need and the build names the rest. + """ + if not args.offline or args.skip_packs: + return + registry = load_data_dir_registry(platforms_dir) + if registry and not any(os.path.isdir(e.get("local_cache") or "") for e in registry.values()): + parser.error( + "no data directory is cached and --offline skips their refresh: run " + "python scripts/refresh_data_dirs.py once, or drop --offline" + ) + + def _truth_steps(args: argparse.Namespace, results: dict) -> bool: """Steps 2c to 2e: truth model, its diff, the native export.""" all_ok = True @@ -723,6 +743,7 @@ def main() -> None: ) args = parser.parse_args() _refuse_unapplied_flags(parser, args) + _refuse_offline_without_caches(parser, args) if not args.offline: # Read once and put in the environment every step inherits: the # scrapers and refresh_data_dirs went anonymous unless the user had diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py index 436e05137..0293bad48 100644 --- a/tests/test_tool_messages.py +++ b/tests/test_tool_messages.py @@ -130,6 +130,34 @@ class TheGapHeadingAddsUp(unittest.TestCase): ) +class AnOfflineRunWithNoCacheStopsFirst(unittest.TestCase): + """On a fresh clone, pipeline.py --offline built every pack before saying + the data directories had never been fetched.""" + + def _args(self, **flags): + import argparse + + return argparse.Namespace(**{"offline": True, "skip_packs": False, **flags}) + + def test_refused_before_any_step(self): + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + cache = Path(tmp) / "cache" + (Path(tmp) / "_data_dirs.yml").write_text( + f"data_directories:\n a:\n local_cache: {cache}\n" + ) + parser = mock.Mock() + pipeline._refuse_offline_without_caches(parser, self._args(), tmp) + self.assertIn("refresh_data_dirs.py", parser.error.call_args.args[0]) + for args in (self._args(offline=False), self._args(skip_packs=True)): + parser = mock.Mock() + pipeline._refuse_offline_without_caches(parser, args, tmp) + parser.error.assert_not_called() + cache.mkdir() + parser = mock.Mock() + pipeline._refuse_offline_without_caches(parser, self._args(), tmp) + parser.error.assert_not_called() + + class TheTruthDiffShowsWhereAndAsksNothing(unittest.TestCase): """The markdown wrote `**Remove** hiscore.dat (phantom)` beside six `**Add** hiscore.dat`, none with a path, for files profiled cores read.""" diff --git a/wiki/tools.md b/wiki/tools.md index e4a19bc10..1acea4c99 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -24,6 +24,10 @@ python scripts/pipeline.py --offline --include-archived # include archived plat python scripts/pipeline.py --check-buildbot # online run + buildbot freshness check ``` +`--offline` skips the data directory refresh, so on a fresh clone run +`python scripts/refresh_data_dirs.py` once first. A run that builds packs with no +data directory cached is refused before step 0. + Pipeline steps: | Step | Description | Skipped by |