feat: report a published export that fell behind

exports/ is committed but CI cannot rebuild it, since the originals it
patches live in the local cache. The native freshness check now runs
the export offline and compares it with the committed files, and
refresh_stale writes the export again after refreshing an original.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 09:28:14 +02:00
1 parent b6a1304893
commit f8e0b33314
4 files changed
+108 -7

No files matched your search

+55 -2
View File
@@ -374,6 +374,48 @@ def native_cache_state(
return ""
def _export_tree(root: Path, platform: str) -> dict[str, bytes]:
"""A platform's published export: its directory and its report."""
tree = {
path.relative_to(root).as_posix(): path.read_bytes()
for path in (root / platform).rglob("*") if path.is_file()
}
report = root / f"{platform}.json"
if report.is_file():
tree[report.name] = report.read_bytes()
return tree
def published_export_state(
platform: str,
exporter_class: type,
truth_dir: Path,
platforms_dir: Path,
cache_dir: Path,
published: Path,
) -> str:
"""Why the committed export is not what the export writes now, '' when it is.
exports/ is committed and the site publishes it, but CI cannot rebuild
it: the originals it patches live in the local cache. A profile change
that moves a hash leaves the published file behind until someone runs
the export again, and this is what says so.
"""
scratch_root = Path("tmp")
scratch_root.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(dir=scratch_root) as scratch:
with contextlib.redirect_stdout(io.StringIO()):
ok, messages = export_native.export_platform(
platform, exporter_class, truth_dir, Path(scratch),
str(platforms_dir), cache_dir, False,
)
if not ok:
return f"the export fails: {messages[0] if messages else 'no message'}"
if _export_tree(Path(scratch), platform) != _export_tree(published, platform):
return f"{published.as_posix()}/{platform} is not what the export writes now"
return ""
def _transcribed_at(platforms_dir: Path, platform: str) -> float:
"""Last rewrite of the platform file or of any file it inherits."""
newest = 0.0
@@ -390,7 +432,10 @@ def _transcribed_at(platforms_dir: Path, platform: str) -> float:
return newest
def check_native(platforms_dir: Path, cache_dir: Path, truth_dir: Path) -> list[Finding]:
def check_native(
platforms_dir: Path, cache_dir: Path, truth_dir: Path,
published: Path = Path("exports"),
) -> list[Finding]:
exporters = discover_exporters()
index = cache_dir / export_native.SOURCES_INDEX
recorded = export_native.load_sources(cache_dir)
@@ -409,13 +454,21 @@ def check_native(platforms_dir: Path, cache_dir: Path, truth_dir: Path) -> list[
reason = native_cache_state(
wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
)
if reason:
reason += ", export_native.py --refresh-cache fetches it"
else:
reason = published_export_state(
name, exporter_class, truth_dir, platforms_dir, cache_dir, published
)
if reason:
reason += f", export_native.py --platform {name} writes it again"
findings.append(
Finding(
"native",
name,
STALE if reason else OK,
local=f"{len(wanted)} file(s)",
detail=f"{reason}, export_native.py --refresh-cache fetches it" if reason else "",
detail=reason,
)
)
return sorted(findings, key=lambda f: f.subject)
+13 -3
View File
@@ -14,7 +14,8 @@ Mapping (stale -> command):
the export patches must be the one just scraped
native/<name> python scripts/generate_truth.py --platform <name>, then
python scripts/export_native.py --platform <name>
--refresh-cache
--refresh-cache, then the export itself, which
rewrites exports/<name>
targets/<name> python -m scripts.scraper.targets.<module>
-o platforms/targets/<name>.yml
data/<key> python scripts/refresh_data_dirs.py --key <key>
@@ -172,9 +173,13 @@ def plan_jobs(
if area == "platforms":
# The truth reads the platform file: regenerated first, or the
# native refresh refuses it as older than the profiles.
then = (tuple(_truth_refresh(subject)), tuple(_native_refresh(subject)))
then = (
tuple(_truth_refresh(subject)),
tuple(_native_refresh(subject)),
tuple(_export_refresh(subject)),
)
elif area == "native":
then = (tuple(_native_refresh(subject)),)
then = (tuple(_native_refresh(subject)), tuple(_export_refresh(subject)))
jobs.append(
Job(area=area, subject=subject, command=command, log_path=log, then=then)
)
@@ -196,6 +201,11 @@ def _native_refresh(platform: str) -> list[str]:
]
def _export_refresh(platform: str) -> list[str]:
"""Rewrite the published export from the refreshed original."""
return [sys.executable, "scripts/export_native.py", "--platform", platform]
def _command_for(
area: str, subject: str, registry: dict[str, dict]
) -> list[str] | None: