From f3a161b60e1e989638d2b9e709c9f23ba58dd238 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Mon, 26 Oct 2026 22:52:23 +0100
Subject: [PATCH] fix: keep unread trees in the source paths record

A sitesources run that GitLab rate-limited dropped about twenty trees it
never read from website/src/data/source-paths.json. A tree key names a
pinned sha, so the last record's entry for an unread tree still holds and
is carried over, in sitesources and in profile_sync's refresh.
---
 scripts/profile_sync.py    |  3 +++
 scripts/sitesources.py     | 24 ++++++++++++++++++++++++
 tests/test_profile_sync.py | 13 +++++++++++++
 3 files changed, 40 insertions(+)

diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py
index 599a798f5..6c5183c82 100644
--- a/scripts/profile_sync.py
+++ b/scripts/profile_sync.py
@@ -4783,6 +4783,9 @@ def _refresh_source_paths(args: argparse.Namespace) -> None:
     held, unread = sitesources.build_source_paths(
         profiles, [sitesources.TREE_CACHE], fetch=True
     )
+    held = sitesources.keep_unread(
+        held, unread, sitesources.read_record(sitesources.SOURCE_PATHS)
+    )
     sitesources.SOURCE_PATHS.parent.mkdir(parents=True, exist_ok=True)
     write_if_changed(
         str(sitesources.SOURCE_PATHS), json.dumps(held, indent=1, sort_keys=True) + "\n"
diff --git a/scripts/sitesources.py b/scripts/sitesources.py
index 065544c47..5974e3c4e 100644
--- a/scripts/sitesources.py
+++ b/scripts/sitesources.py
@@ -313,6 +313,29 @@ def build_source_paths(
     return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread)
 
 
+def keep_unread(
+    held: dict[str, list[str]], unread: list[str], previous: dict[str, list[str]]
+) -> dict[str, list[str]]:
+    """Carry over what the last record held for the trees this run could not read.
+
+    A tree key names a pinned sha, so what it held then it holds now. A run
+    that GitLab rate-limited dropped about twenty trees it never read.
+    """
+    kept = dict(held)
+    for key in unread:
+        if key not in kept and key in previous:
+            kept[key] = previous[key]
+    return dict(sorted(kept.items()))
+
+
+def read_record(path: Path) -> dict[str, list[str]]:
+    """The record a previous run wrote, empty when there is none."""
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return {}
+
+
 def _commit_url(repo: upstream.Repo, pin: str) -> str:
     """Forge-specific browser URL for one commit."""
     base = f"https://{repo.host}/{repo.owner}/{repo.name}"
@@ -662,6 +685,7 @@ def main() -> int:
 
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
     held, unread = build_source_paths(profiles, cache_dirs, args.fetch)
+    held = keep_unread(held, unread, read_record(args.output))
     args.output.parent.mkdir(parents=True, exist_ok=True)
     write_if_changed(str(args.output), json.dumps(held, indent=1, sort_keys=True) + "\n")
     print(f"{sum(map(len, held.values()))} paths held in {len(held)} trees, "
diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py
index 7b98f624c..bf875005c 100644
--- a/tests/test_profile_sync.py
+++ b/tests/test_profile_sync.py
@@ -4667,6 +4667,19 @@ class AWriteRefreshesTheSourcePaths(unittest.TestCase):
             profile_sync._WRITTEN.clear()
 
 
+class AnUnreadTreeKeepsItsRecord(unittest.TestCase):
+    """sitesources --fetch, rate-limited by GitLab, dropped about twenty
+    trees it never read from source-paths.json."""
+
+    def test_unread_trees_carry_over(self):
+        import sitesources  # noqa: PLC0415
+
+        kept = sitesources.keep_unread(
+            {"a@1": ["x.c"]}, ["b@2", "c@3"], {"b@2": ["y.c"], "a@1": ["old.c"]}
+        )
+        self.assertEqual(kept, {"a@1": ["x.c"], "b@2": ["y.c"]})
+
+
 class FreshRefusesOffline(unittest.TestCase):
     def test_the_pair_is_refused(self):
         saved = sys.argv