diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 2c2f23ede..90976b050 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -883,6 +883,32 @@ def _refuse_offline_without_caches( ) +def _refuse_offline_export_without_originals( + parser: argparse.ArgumentParser, args: argparse.Namespace, + cache: str = ".cache/upstream-native", +) -> None: + """An offline run with --with-export needs each patched file cached. + + Seven formats carry code, so step 2e patches the platform's own file, + which --offline does not fetch. On a fresh clone the database, verify + and the truth model were built before step 2e failed on every one. + """ + if not (args.offline and args.with_export): + return + from exporter import discover_exporters # noqa: PLC0415 + + lacking = sorted( + name for name, exporter in discover_exporters().items() + if exporter.needs_original() and not any(Path(cache, name).glob("*")) + ) + if lacking: + parser.error( + f"--with-export patches each platform's own file, and {cache}/ holds none " + f"for {', '.join(lacking)}: run python scripts/export_native.py --all " + "--refresh-cache once, or drop --offline" + ) + + def _truth_steps(args: argparse.Namespace, results: dict) -> bool: """Steps 2c to 2e: truth model, its diff, the native export.""" all_ok = True @@ -1010,6 +1036,7 @@ def main() -> None: args = parser.parse_args() _refuse_unapplied_flags(parser, args) _refuse_offline_without_caches(parser, args) + _refuse_offline_export_without_originals(parser, args) if not args.offline: # Read once and put in the environment every step inherits: the # scrapers and refresh_data_dirs went anonymous unless the user had diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py index 7f043a22b..3f1a8abbd 100644 --- a/tests/test_tool_messages.py +++ b/tests/test_tool_messages.py @@ -175,6 +175,44 @@ class AnOfflineRunWithNoCacheStopsFirst(unittest.TestCase): parser.error.assert_not_called() +class AnOfflineExportWithNoOriginalsStopsFirst(unittest.TestCase): + """pipeline.py --offline --with-export, which tools.md listed as a + standard command, built the database, verify and the truth model on a + fresh clone, then failed step 2e on every platform whose own file it + patches: --offline never passes --fetch.""" + + def _args(self, **flags): + import argparse + + return argparse.Namespace(**{"offline": True, "with_export": True, **flags}) + + def test_refused_before_any_step(self): + from exporter import discover_exporters + + patched = sorted(n for n, e in discover_exporters().items() if e.needs_original()) + self.assertIn("batocera", patched) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + parser = mock.Mock() + pipeline._refuse_offline_export_without_originals(parser, self._args(), tmp) + message = parser.error.call_args.args[0] + self.assertIn("export_native.py --all --refresh-cache", message) + self.assertIn("batocera", message) + for args in (self._args(offline=False), self._args(with_export=False)): + parser = mock.Mock() + pipeline._refuse_offline_export_without_originals(parser, args, tmp) + parser.error.assert_not_called() + for name in patched: + (Path(tmp) / name).mkdir() + (Path(tmp) / name / "original").write_text("x") + parser = mock.Mock() + pipeline._refuse_offline_export_without_originals(parser, self._args(), tmp) + parser.error.assert_not_called() + + def test_the_tools_page_says_so(self): + text = (REPO_ROOT / "wiki" / "tools.md").read_text(encoding="utf-8") + self.assertIn("export_native.py --all --refresh-cache", text) + + class TheTruthDiffShowsWhereAndAsksNothing(unittest.TestCase): """The markdown wrote `**Remove** hiscore.dat (phantom)` beside six `**Add** hiscore.dat`, none with a path, for files profiled cores read.""" diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index c17cc07ac..d54969839 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -534,7 +534,8 @@ Include truth generation and diffing in the pipeline: python scripts/pipeline.py --offline --with-truth ``` -Include truth + native format export: +Include truth + native format export. Offline, the platforms' own files +must already be cached ([tools](tools.md#pipeline)): ```bash python scripts/pipeline.py --offline --with-export diff --git a/wiki/tools.md b/wiki/tools.md index c7dfae1dc..2c4b4290a 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -28,6 +28,13 @@ python scripts/pipeline.py --check-buildbot # online run + buildbot fresh `python scripts/refresh_data_dirs.py` once first. A run that builds packs with no data directory cached is refused before step 0. +`--with-export` online passes `--fetch` to step 2e, which downloads each +platform's own file and patches it. Offline that file must already be in +`.cache/upstream-native/`: on a fresh clone run +`python scripts/export_native.py --all --refresh-cache` once first. An offline +run with `--with-export` and an empty cache for any platform it patches is +refused before step 0. + Pipeline steps: | Step | Description | Skipped by |