diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index d12d251b..7f582294 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -399,7 +399,11 @@ def check_native(platforms_dir: Path, cache_dir: Path, truth_dir: Path) -> list[ exporter_class = exporters.get(name) if not exporter_class: continue - truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir)) + try: + truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir)) + except export_native.StaleTruth as exc: + findings.append(Finding("native", name, STALE, detail=str(exc))) + continue systems, _ = build_native_model(truth or {}, scraped) wanted = export_native.wanted_sources(exporter_class(), systems, scraped) reason = native_cache_state( diff --git a/scripts/export_native.py b/scripts/export_native.py index 9ef4a8a5..5a623b67 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -265,8 +265,11 @@ class StaleTruth(ValueError): @functools.lru_cache(maxsize=None) def _truth_sources(platforms_dir: str) -> tuple[dict, dict, dict | None]: """Registry, profiles and database, as generate_truth reads them.""" - with open(Path(platforms_dir) / "_registry.yml") as handle: - registry = (yaml_load(handle) or {}).get("platforms", {}) + registry_path = Path(platforms_dir) / "_registry.yml" + registry: dict = {} + if registry_path.exists(): + with open(registry_path) as handle: + registry = (yaml_load(handle) or {}).get("platforms", {}) profiles = load_emulator_profiles("emulators") db = load_database(TRUTH_DB) if Path(TRUTH_DB).exists() else None return registry, profiles, db diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index 4aa643ab..c9ebb363 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -12,7 +12,8 @@ Mapping (stale -> command): -o platforms/.yml then the native refresh below: the original the export patches must be the one just scraped - native/ python scripts/export_native.py --platform + native/ python scripts/generate_truth.py --platform , then + python scripts/export_native.py --platform --refresh-cache targets/ python -m scripts.scraper.targets. -o platforms/targets/.yml @@ -169,6 +170,10 @@ def plan_jobs( log = LOG_DIR / f"{area}__{_slug(subject)}.log" then: tuple[tuple[str, ...], ...] = () if area == "platforms": + # The truth reads the platform file: regenerated first, or the + # native refresh refuses it as older than the profiles. + then = (tuple(_truth_refresh(subject)), tuple(_native_refresh(subject))) + elif area == "native": then = (tuple(_native_refresh(subject)),) jobs.append( Job(area=area, subject=subject, command=command, log_path=log, then=then) @@ -177,6 +182,10 @@ def plan_jobs( return jobs, surfaced, ignored +def _truth_refresh(platform: str) -> list[str]: + return [sys.executable, "scripts/generate_truth.py", "--platform", platform] + + def _native_refresh(platform: str) -> list[str]: return [ sys.executable, @@ -192,7 +201,9 @@ def _command_for( ) -> list[str] | None: """Map a stale finding to its refresh command, or None if manual.""" if area == "native": - return _native_refresh(subject) + # export_native refuses a truth older than the profiles, so the + # truth comes first and the original follows (Job.then). + return _truth_refresh(subject) if area == "platforms": entry = registry.get(subject) or {} diff --git a/tests/test_export_native_layer.py b/tests/test_export_native_layer.py index 1cbcdb91..6e93642d 100644 --- a/tests/test_export_native_layer.py +++ b/tests/test_export_native_layer.py @@ -48,7 +48,7 @@ class TheNativeLayerIsThePlatformsOwn(unittest.TestCase): " curated: true\n" ) pack_view = common.load_platform_config("demo", str(platforms)) - _truth, native = load_inputs("demo", platforms, str(platforms)) + _truth, native = load_inputs("demo", platforms / "truth", str(platforms)) self.assertEqual( [f["name"] for f in pack_view["systems"]["nec-pc-88"]["files"]], diff --git a/tests/test_refresh_stale.py b/tests/test_refresh_stale.py index e647088e..42ecb139 100644 --- a/tests/test_refresh_stale.py +++ b/tests/test_refresh_stale.py @@ -41,9 +41,12 @@ class Planning(unittest.TestCase): jobs[0].command[1:], ["-m", "scripts.scraper.retrobat_scraper", "-o", "platforms/retrobat.yml"], ) + # The truth reads the platform file and export_native refuses one + # older than the profiles: regenerated between the two. self.assertEqual( [list(c[1:]) for c in jobs[0].then], - [["scripts/export_native.py", "--platform", "retrobat", "--refresh-cache"]], + [["scripts/generate_truth.py", "--platform", "retrobat"], + ["scripts/export_native.py", "--platform", "retrobat", "--refresh-cache"]], ) def test_a_native_row_of_a_rescraped_platform_is_not_a_second_job(self): @@ -57,10 +60,12 @@ class Planning(unittest.TestCase): def test_a_stale_original_alone_is_refreshed(self): jobs, _, _ = self._plan(finding("native", "lakka")) self.assertEqual( - jobs[0].command[1:], - ["scripts/export_native.py", "--platform", "lakka", "--refresh-cache"], + jobs[0].command[1:], ["scripts/generate_truth.py", "--platform", "lakka"] + ) + self.assertEqual( + [list(c[1:]) for c in jobs[0].then], + [["scripts/export_native.py", "--platform", "lakka", "--refresh-cache"]], ) - self.assertEqual(jobs[0].then, ()) def test_a_platform_without_a_scraper_is_surfaced(self): jobs, surfaced, _ = self._plan(finding("platforms", "lakka"))