From 64b2967ffbe83f41376ec41f1e4823897f4a0931 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:14:47 +0200 Subject: [PATCH 1/5] fix: read a source_ref role per part, prose too The role a part writes (source or upstream) was kept per path, first met first kept, so a later part naming the other role was read in the other repository. In prose the word source made a citation external and the word upstream was dropped. The role now travels on each part. --- scripts/profile_sync.py | 100 +++++++++++++++++++++++++++--------- scripts/sitesources.py | 4 ++ scripts/sourceref.py | 22 +++++++- tests/test_profile_sync.py | 102 +++++++++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 27 deletions(-) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 67ae045c..378729c9 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -1305,12 +1305,22 @@ def _resolve_project_words( if part.path not in resolved: resolved[part.path] = resolve_project_word(part.path, tree_order()) if resolved[part.path]: - part = RefPart(resolved[part.path], part.start, part.end, part.raw) + part = replace(part, path=resolved[part.path]) rewritten.append(part) out.append((display, ref, tokens, hashes, rewritten, citation)) return out +def _path_roles(refs: list) -> dict[str, set[str]]: + """The roles each cited path is written under across the profile.""" + roles: dict[str, set[str]] = {} + for *_, parts, _citation in refs: + for part in parts: + if part.role: + roles.setdefault(part.path, set()).add(part.role) + return roles + + def build_report( name: str, profile: dict, cache_dir: str, offline: bool = False, text: str | None = None, @@ -1328,20 +1338,20 @@ def build_report( if (repo := upstream.parse_repo(url)) is not None ) refs = [] - # The repository a part says its lines are in (`upstream src/a.c:5`). - cited_roles: dict[str, str] = {} for citation in collect_citations(profile, text): if citation.kind == "ref": - for p in split_source_ref(citation.ref): - role, _bare = ref_role(p.path) - if role: - cited_roles.setdefault(strip_repo_word(p.path, known), role) entry = citation.entry display = str(entry.get("name") or "") or citation.field if citation.label: display += f" [{citation.label}]" + # The repository a part says its lines are in (`upstream + # src/a.c:5`) stays on that part: the same path can be cited + # under both roles in one profile. parts = [ - RefPart(strip_repo_word(p.path, known), p.start, p.end, p.raw) + RefPart( + strip_repo_word(p.path, known), p.start, p.end, p.raw, + ref_role(p.path)[0], + ) for p in split_source_ref(citation.ref) ] refs.append(( @@ -1353,6 +1363,19 @@ def build_report( citation.field, citation.ref, [], [], list(citation.parts), citation, )) + # A part that names no repository is looked up first where its path is + # cited elsewhere: a note citing `src/a.c:12` beside source_refs that + # all read `upstream src/a.c` means that tree. + # A path cited under both roles lends neither, and such a part is + # compared in every repository rather than in whichever comes first. + path_roles = _path_roles(refs) + + def role_of(part: RefPart) -> str: + if part.role: + return part.role + lent = path_roles.get(part.path, ()) + return next(iter(lent)) if len(lent) == 1 else "" + # Directories the profile already cites, deepest first, to settle a bare # filename that several trees of one repository carry. cited_dirs: set[str] = set() @@ -1448,7 +1471,7 @@ def build_report( pin_trees: dict[str, list[str] | None] = {} def _locate( - path: str, start: int | None = None, tokens: tuple = () + path: str, start: int | None = None, tokens: tuple = (), wanted: str = "" ) -> tuple[RepoView, str]: """Repository and real path carrying a cited path, HEAD before pin. @@ -1539,7 +1562,6 @@ def build_report( return 1 if line.strip() else 0 # A part that names its repository is looked up there first. - wanted = cited_roles.get(path, "") ordered = sorted(views, key=lambda v: _view_role(v) != wanted) if wanted else views best = None for candidate in candidates: @@ -1572,11 +1594,11 @@ def build_report( return (best[1], best[2]) if best else (primary, path) def resolve_path( - path: str, start: int | None = None, tokens: tuple = () + path: str, start: int | None = None, tokens: tuple = (), role: str = "" ) -> tuple[RepoView, str]: - key = (path, start, tokens) + key = (path, start, tokens, role) if key not in owners: - owners[key] = _locate(path, start, tokens) + owners[key] = _locate(path, start, tokens, role) return owners[key] def _pin_tree_for(view: RepoView) -> list[str] | None: @@ -1634,14 +1656,14 @@ def build_report( ) return context[key] - def rename_getter(path: str) -> tuple[str | None, list[str]]: + def rename_getter(path: str, role: str = "") -> tuple[str | None, list[str]]: """Resolved only when a cited file has vanished, then memoised. Every declared repository is searched, not just the one that owns the path: a profile citing both a port and its upstream may point at a file that only the other one carries. """ - owner, actual = resolve_path(path) + owner, actual = resolve_path(path, role=role) ordered = [owner] + [v for v in views if v is not owner] candidates: list[str] = [] for view in ordered: @@ -1673,6 +1695,7 @@ def build_report( start: int | None = None, tokens: Iterable[str] = (), forced: RepoView | None = None, + role: str = "", ) -> list[str] | None: """Lines for one path at one revision, read once per run. @@ -1680,10 +1703,11 @@ def build_report( on identity. """ tokens = tuple(tokens) - key = (which, path, start, tokens, forced and forced.repo.slug) + key = (which, path, start, tokens, forced and forced.repo.slug, role) if key not in lines_cache: view, actual = ( - (forced, path) if forced else resolve_path(path, start, tokens) + (forced, path) if forced + else resolve_path(path, start, tokens, role) ) sha = view.pin if which == PIN else view.head found = fetch_from(view.repo, sha, actual) @@ -1701,9 +1725,11 @@ def build_report( start: int | None = None, tokens: Iterable[str] = (), forced: RepoView | None = None, + role: str = "", ) -> tuple[str | None, str, str]: view, actual = ( - (forced, path) if forced else resolve_path(path, start, tuple(tokens)) + (forced, path) if forced + else resolve_path(path, start, tuple(tokens), role) ) slug = view.repo.slug if view is not primary else None return slug, upstream.raw_url(view.repo, view.head, actual), actual @@ -1715,12 +1741,20 @@ def build_report( both mamedev/mame and its libretro fork hold, with the same delimited token on the cited line, so no single attribution can be right for all of them. Taking the best outcome across repositories settles it. + A part that names its repository is judged there alone when that + repository carries the path: hypseus-singe cites lair.cpp under + `source` for one entry and `upstream` for the next. """ - owner, _actual = resolve_path(part.path, part.start, tuple(tokens)) + owner, _actual = resolve_path( + part.path, part.start, tuple(tokens), role_of(part) + ) carrying = [ view for view in views if fetch_from(view.repo, view.pin, part.path) is not None ] + named = [view for view in carrying if _view_role(view) == part.role] + if part.role and named: + carrying = named live = { id(view) for view in carrying if fetch_from(view.repo, view.head, part.path) is not None @@ -1732,7 +1766,7 @@ def build_report( lambda which, path, start=None, toks=(), _v=view: fetch( which, path, start, toks, _v ), - rename_getter, + lambda path: rename_getter(path, role_of(part)), lambda path, start=None, toks=(), _v=view: describe( path, start, toks, _v ), @@ -1753,7 +1787,15 @@ def build_report( if best is None or rank < best[0] or (rank == best[0] and view is owner): best = (rank, result) return best[1] if best else anchor_part( - part, fetch, rename_getter, describe, tokens + part, + lambda which, path, start=None, toks=(): fetch( + which, path, start, toks, role=role_of(part) + ), + lambda path: rename_getter(path, role_of(part)), + lambda path, start=None, toks=(): describe( + path, start, toks, role=role_of(part) + ), + tokens, ) # Comparing a revision with itself anchors every ref whatever it cites, so @@ -1763,7 +1805,9 @@ def build_report( def self_checked(part: RefPart, tokens: Sequence[str]) -> bool: if report.pinned_tag: return True - view, _ = resolve_path(part.path, part.start, tuple(tokens)) + if not part.role and len(path_roles.get(part.path, ())) > 1: + return all(view.pin == view.head for view in views) + view, _ = resolve_path(part.path, part.start, tuple(tokens), role_of(part)) return view.pin == view.head def judge( @@ -1779,11 +1823,15 @@ def build_report( ) if flag: result = verify_at_pin( - part, fetch(PIN, part.path, part.start, tokens), tokens, hashes + part, + fetch(PIN, part.path, part.start, tokens, role=role_of(part)), + tokens, hashes, ) # Checked at its own revision, the part still lies in one tree: # the report names it, as a comparison does. - owner, _actual = resolve_path(part.path, part.start, tuple(tokens)) + owner, _actual = resolve_path( + part.path, part.start, tuple(tokens), role_of(part) + ) if result.repo is None and owner is not primary: result = replace(result, repo=owner.repo.slug) else: @@ -1791,7 +1839,9 @@ def build_report( # A forge that refused the request said nothing about the file: # git.eden-emu.dev answers 403 to scripts, and nine citations of # eden-nightly read "absent at HEAD" for a file that exists. - owner, _actual = resolve_path(part.path, part.start, tuple(tokens)) + owner, _actual = resolve_path( + part.path, part.start, tuple(tokens), role_of(part) + ) if ( result.status in ("GONE", "AMBIGUOUS") and (owner.repo.host, owner.repo.slug) in mute diff --git a/scripts/sitesources.py b/scripts/sitesources.py index 9833c2bb..7d9e0912 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -261,6 +261,10 @@ def link_prose_citations(profile: dict, text: str) -> str: continue if "/" not in path and path not in structured: continue + role = parts[0].role + if role and (role_sources(profile).get(role) or (None,))[0] != repo: + # `upstream a.c:5` names a repository this link would not reach. + continue if any(lo <= path_span[0] < hi for lo, hi in guarded): continue for index, ((lo, hi), part) in enumerate(zip(spans, parts)): diff --git a/scripts/sourceref.py b/scripts/sourceref.py index 598085f2..9ec48688 100644 --- a/scripts/sourceref.py +++ b/scripts/sourceref.py @@ -39,6 +39,10 @@ class RefPart: start: int | None end: int | None raw: str = "" + # The repository the part names (`upstream src/a.c:5`), empty when it + # names none. Carried per part: one profile cites the same path in both + # repositories, hypseus-singe's lair.cpp under `source` and `upstream`. + role: str = "" @property def last(self) -> int | None: @@ -247,6 +251,19 @@ def _prose_external( return not lead or lead[-1] in _PROSE_LIST_DELIMITERS +def _prose_role(line: str, path_start: int) -> str: + """The role word written right before a prose path, empty when none. + + Prose follows the source_ref convention: `(upstream retro_host.c:196)` + names the original emulator, `ref: source src/devices/floppy.cpp:1656` + the repository declared as source. Read as a linking word or a project + name, the word was dropped or made the citation external. + """ + before = line[:path_start].rstrip() + word = before[max(before.rfind(" "), before.rfind("\t"), before.rfind("(")) + 1:] + return word if word in REPO_ROLES else "" + + def resolve_project_word(path: str, trees: Iterable[list[str] | None]) -> str | None: """Path of `project file` inside a declared tree, None when out of reach. @@ -333,7 +350,8 @@ def _prose_runs( ): path_start -= 1 path = "." + path - if _prose_external(line, path_start, known): + role = _prose_role(line, path_start) + if not role and _prose_external(line, path_start, known): # Kept with its project word, the source_ref convention: the # report looks for the project inside the declared trees # (mupen64plus-ae vendors `rice Config.cpp`) and calls the @@ -358,7 +376,7 @@ def _prose_runs( for lo, hi in spans: rng = line[lo:hi] a, _, b = rng.partition("-") - parts.append(RefPart(path, int(a), int(b or a), rng)) + parts.append(RefPart(path, int(a), int(b or a), rng, role)) yield ( number, path_span, tuple(spans), tuple(parts), line[path_start:end], end > boundary, diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 60e09cfe..1dbc8b29 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -4340,5 +4340,107 @@ class EachCitationNamesTheCodeThatAsks(unittest.TestCase): self.assertEqual(self._attribute("source emu/memory.c:2"), "source emu/memory.c:2") +class ARoleIsReadPerPart(unittest.TestCase): + """The role a part writes (`source a.c:3`) was stored per path, first met + first kept: hypseus-singe's `upstream lair.cpp` decided where every later + `source lair.cpp` was read, and correct refs were reported MOVED.""" + + setUp = TestBuildReport.setUp + tearDown = TestBuildReport.tearDown + MD5A = "0123456789abcdef0123456789abcdef" + MD5B = "fedcba9876543210fedcba9876543210" + + def _report(self, files: str, notes: str = "") -> ProfileReport: + # The port inserted two lines above its copy of the table since its + # pin, while the original sits on HEAD with the same table. + trees = { + ("libretro/x", "pins"): {"a.c": ["pad", "pad", f"rom {self.MD5B}"]}, + ("libretro/x", "heads"): {"a.c": ["pad", "pad", "new", "new", + f"rom {self.MD5B}"]}, + ("alek/x", "pinu"): {"a.c": [f"rom {self.MD5A}", "x", + f"rom {self.MD5B}"]}, + } + profile_sync.upstream.fetch_file = ( + lambda repo, sha, path, cache_dir, offline=False: + trees.get((repo.slug, sha), {}).get(path) + ) + profile_sync.upstream.list_tree = ( + lambda repo, sha, cache_dir, offline=False: + (sorted(trees.get((repo.slug, sha), {})), False) + ) + profile_sync.upstream.resolve_head = ( + lambda repo, cache_dir, offline=False, branch=None: + "heads" if repo.owner == "libretro" else "pinu" + ) + profile_sync.upstream.compare = ( + lambda repo, base, head, cache_dir, offline=False: + profile_sync.CompareResult([], False) + ) + text = ( + 'emulator: T\n' + 'source: "https://github.com/libretro/x"\n' + 'upstream: "https://github.com/alek/x"\n' + 'source_commit: "pins"\n' + 'upstream_commit: "pinu"\n' + + (f"notes: |\n {notes}\n" if notes else "") + + "files:\n" + files + ) + profile = yaml.safe_load(text) + with contextlib.redirect_stderr(io.StringIO()): + return build_report("t", profile, self.dir, text=text) + + def test_a_part_naming_its_repository_is_judged_there(self): + report = self._report( + " - name: a.rom\n" + f" md5: {self.MD5A}\n" + ' source_ref: "upstream a.c:1"\n' + " - name: b.rom\n" + f" md5: {self.MD5B}\n" + ' source_ref: "source a.c:3"\n' + ) + parts = {e.name: e.parts[0] for e in report.entries} + self.assertEqual(parts["a.rom"].status, "ANCHORED") + self.assertEqual(parts["a.rom"].repo, "alek/x") + # Read in the original, where line 3 still holds the table, the + # port's own move went unseen. + self.assertEqual(parts["b.rom"].status, "SHIFTED") + self.assertEqual(parts["b.rom"].start, 5) + self.assertIsNone(parts["b.rom"].repo) + + def test_a_prose_source_word_is_a_role_not_a_project(self): + report = self._report( + " - name: b.rom\n" + f" md5: {self.MD5B}\n" + ' source_ref: "source a.c:3"\n', + notes="ref: source a.c:3, more", + ) + prose = [e for e in report.entries if e.kind == "prose"] + self.assertEqual(len(prose), 1) + self.assertNotEqual(prose[0].parts[0].status, "EXTERNAL") + self.assertEqual(prose[0].parts[0].part.role, "source") + + def test_a_prose_upstream_word_names_the_repository(self): + report = self._report( + " - name: b.rom\n" + f" md5: {self.MD5B}\n" + ' source_ref: "source a.c:3"\n', + notes="the original reads it (upstream a.c:1).", + ) + prose = [e for e in report.entries if e.kind == "prose"] + self.assertEqual(prose[0].parts[0].part.role, "upstream") + self.assertEqual(prose[0].parts[0].repo, "alek/x") + + def test_the_prose_grammar_keeps_the_role(self): + from sourceref import _prose_runs + + runs = list(_prose_runs( + "# ref: source src/a.cpp:1656-1685, 1709-1738, upstream b.c:5" + )) + roles = [(p.path, p.role) for run in runs for p in run[3]] + self.assertEqual(roles, [ + ("src/a.cpp", "source"), ("src/a.cpp", "source"), ("b.c", "upstream"), + ]) + + if __name__ == "__main__": unittest.main() From 31566ce618b80f20650426d7bd6776d3d1cefb2b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:17:01 +0200 Subject: [PATCH 2/5] fix: read a past pin only for its own repository A source_commit string from before the field was keyed by mode pinned the standalone repository, yet was read as a writing pin for every mode. xenia-edge then saw its libretro pin move under unchanged prose, and the recale and bump were refused. --- scripts/profile_sync.py | 44 ++++++++++++++++++++++++++++++++------ tests/test_profile_sync.py | 22 +++++++++++++++++++ 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 378729c9..b7459194 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -2877,12 +2877,41 @@ def _scalar_values(document: dict) -> dict[str, str]: } -def _pin_of(document: dict, field: str, mode: str = "") -> str | None: - """A pin as the profile records it, read for one build mode when keyed.""" +def _pin_of( + document: dict, field: str, mode: str = "", + repo: upstream.Repo | None = None, +) -> str | None: + """A pin as the profile records it, read for one build mode when keyed. + + With `repo`, a pin the document declares for another repository under + the field and mode does not count. A past revision can pin another + repository under the same field: xenia-edge's source_commit named the + has207/xenia-edge sha until the field became a dict keyed by mode, and + read for the libretro mode that old string passed for a pin of the + GitLab port that moved under unchanged prose. + """ value = document.get(field) if isinstance(value, dict): value = value.get(mode) if mode else None - return value if isinstance(value, str) and value else None + if not (isinstance(value, str) and value): + return None + if repo is not None: + url_field = field.removesuffix("_commit") + declared = { + (mode_key, url) + for name, mode_key, url in declared_repositories(document) + if name == url_field + } + url = next( + (u for m, u in declared if m == mode), + next((u for m, u in declared if not m), None), + ) + owner = upstream.parse_repo(url) if url else None + if owner is not None and (owner.host, owner.slug.lower()) != ( + repo.host, repo.slug.lower() + ): + return None + return value _CITATION_RUN_RE = re.compile( @@ -2906,7 +2935,8 @@ def _scalar_citations(text: str | None) -> tuple[str, ...] | None: def _writing_pins( - revisions: list[tuple[str, dict]], intro_sha: str, field: str, mode: str = "" + revisions: list[tuple[str, dict]], intro_sha: str, field: str, mode: str = "", + repo: upstream.Repo | None = None, ) -> list[str]: """Distinct pins committed while the scalar held its current text. @@ -2925,7 +2955,7 @@ def _writing_pins( return [] pins: list[str] = [] for position in range(index, -1, -1): - value = _pin_of(revisions[position][1], field, mode) + value = _pin_of(revisions[position][1], field, mode, repo) if value and value not in pins: pins.append(value) return pins @@ -3038,7 +3068,9 @@ def _writing_pairs( current = _pin_of(document, f"{pin_field}_commit", mode) if not current: continue - written = _writing_pins(revisions, intro_sha, f"{pin_field}_commit", mode) + written = _writing_pins( + revisions, intro_sha, f"{pin_field}_commit", mode, repo + ) if attested in written: written = [attested] if len(written) > 1: diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 1dbc8b29..ebfedead 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -2473,6 +2473,28 @@ class TestWriteDryRun(unittest.TestCase): ) self.assertIn([(repo, "old", "new")], readings) + def test_a_pin_of_another_repository_is_no_writing_pin(self): + """xenia-edge: source_commit named the has207/xenia-edge sha until it + became a dict whose libretro key pins a GitLab port. Read for that + mode, the old string passed for a pin of the port that moved under + unchanged prose, and the bump stayed refused for good.""" + edge = "https://github.com/has207/xenia-edge" + port = "https://gitlab.com/x/xenia-libretro" + modes = {"standalone": edge, "libretro": port} + revisions = [ + ("c2", {"source": modes, "notes": "x", + "source_commit": {"standalone": "e1", "libretro": "p1"}}), + ("c1", {"source": edge, "notes": "x", "source_commit": "e1"}), + ] + document = dict(revisions[0][1]) + repos = [("source", mode, profile_sync.upstream.parse_repo(url)) + for mode, url in modes.items()] + readings, ambiguous = profile_sync._writing_pairs( + document, repos, revisions, "c1" + ) + self.assertEqual(readings, [[]]) + self.assertEqual(ambiguous, []) + def test_stale_prose_holds_the_pin(self): """boom3: a note written at an older pin is not anchored from this one.""" before = self.path.read_text() From 805ca3bb6b919be47a8cbe12e75b62b05d6143f3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:18:49 +0200 Subject: [PATCH 3/5] fix: pass over a profile revision that won't parse One unparsable commit in a profile's history stopped the walk and marked every older prose citation unknown, which refused the recale and bump for good (lindbergh-loader, df7346b3). The revision is now passed over unless the pins differ on its two sides while the field's citations changed across it. --- scripts/profile_sync.py | 29 +++++++++++++++++++++++++-- tests/test_profile_sync.py | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index b7459194..b0de1ba8 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -3178,13 +3178,34 @@ def realign_prose( # Fields whose writing pin lies past a revision that cannot be read: # where their text came in is unknown, so they are not moved. unknown: dict[str, str] = {} + # A revision that does not parse is passed over unless the pins differ + # on its two sides while the field's citations changed across it: then + # the text may have come in there under either pin. Otherwise nothing + # it could hold changes the reading. lindbergh-loader + # committed invalid YAML once (df7346b3), fixed the same day, and every + # note older than that commit stayed blocked for good. + gap: str | None = None for sha in history: if not any(alive.values()): break past = _git_file_at(path, sha) if past is None: - unknown = {field: sha for field, live in alive.items() if live} - break + gap = gap or sha + continue + if gap: + newer = revisions[-1][1] if revisions else document + moved = any( + newer.get(pin) != past.get(pin) + for pin in ("source_commit", "upstream_commit") + ) + values = _scalar_values(past) + for field, live in alive.items(): + if live and moved and _scalar_citations(values.get(field)) != ( + _scalar_citations(current_values.get(field)) + ): + unknown[field] = gap + alive[field] = False + gap = None revisions.append((sha, past)) values = _scalar_values(past) for field in fields: @@ -3196,6 +3217,10 @@ def realign_prose( intro[field] = sha else: alive[field] = False + if gap: + # The oldest revisions do not parse: where the text came in is past + # what can be read. + unknown.update({field: gap for field, live in alive.items() if live}) messages: list[str] = [] jobs: list[tuple[Citation, dict]] = [] diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index ebfedead..aa852bd5 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -3845,6 +3845,46 @@ class TestRealignProse(unittest.TestCase): self.assertTrue(all(m.startswith("read again: notes:") for m in messages)) self.assertIn("(a.c:10)", self.path.read_text()) + def _repo_with_a_broken_revision(self, broken_pin: str): + self._git("init", "-q") + self._git("config", "user.email", "t@t") + self._git("config", "user.name", "t") + written = PROSE_SAMPLE.replace('"pin"', '"oldpin"') + self._commit(written, "profile") + broken = PROSE_SAMPLE.replace('"pin"', f'"{broken_pin}"') + self._commit(broken + "files: [\n", "invalid yaml") + self._commit(written, "repair") + self.path.write_text(PROSE_SAMPLE, encoding="utf-8") + self.files[("oldpin", "a.c")] = ["x"] * 9 + ["subject"] + self.files[("pin", "a.c")] = ["x"] * 13 + ["subject"] + self.files[("oldpin", "b.c")] = list("pqrstuvwx") + self.files[("pin", "b.c")] = list("pqrstuvwx") + + def test_a_revision_that_does_not_parse_is_passed_over(self): + """lindbergh-loader committed invalid YAML once, repaired the same + day under the same pin. Every older note was reported unreadable + and the recale stayed refused for good.""" + self._repo_with_a_broken_revision("oldpin") + messages = profile_sync.realign_prose(self.path, self.tmp.name, dry_run=True) + self.assertFalse(any("unreadable" in m for m in messages), messages) + self.assertTrue(any("would recale" in m for m in messages), messages) + + def test_a_revision_that_does_not_parse_still_blocks_when_it_can_matter(self): + """The pins differ across the gap and the note's citations changed: + the text may have come in there under either pin.""" + self._git("init", "-q") + self._git("config", "user.email", "t@t") + self._git("config", "user.name", "t") + self._commit( + PROSE_SAMPLE.replace('"pin"', '"older"').replace("a.c:10", "a.c:3"), + "profile", + ) + self._commit(PROSE_SAMPLE + "files: [\n", "invalid yaml") + self._commit(PROSE_SAMPLE.replace('"pin"', '"oldpin"'), "repair") + self.path.write_text(PROSE_SAMPLE, encoding="utf-8") + messages = profile_sync.realign_prose(self.path, self.tmp.name, dry_run=True) + self.assertTrue(any("unreadable" in m for m in messages), messages) + def test_scalar_written_at_the_current_pin_is_left_alone(self): self._git("init", "-q") self._git("config", "user.email", "t@t") From b464ed9202bfb894894825566698f15865eaea37 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:24:17 +0200 Subject: [PATCH 4/5] fix: count prose that holds the pin as review work The triage, --changed-only and --strict counted refs only. Prose that --realign-prose would move or refuse blocks every recale and bump, and blastem read 0 to review. Each report now carries that reading. The triage also gets a header, a closing count, and one sampling notice per run instead of one per profile. --- scripts/profile_sync.py | 82 +++++++++++++++++++++++++++++++++----- tests/test_profile_sync.py | 66 ++++++++++++++++++++++++++++++ wiki/tools.md | 10 ++++- 3 files changed, 147 insertions(+), 11 deletions(-) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index b0de1ba8..968e42ad 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -945,6 +945,11 @@ class ProfileReport: page_version: str | None = None # An announced version searched for and found only rebuilt or re-signed. no_genuine_copy: str | None = None + # What realign_prose has to settle before a recale or a bump can run: + # prose written under an older pin. None when not looked at. Counted to + # review, or the triage read "0 to review" on a profile whose pin could + # not move (blastem, xenia-edge, lindbergh-loader). + stale_prose: list[str] | None = None # Skips that say the profile cannot be checked as written: a repository # that no longer answers, no pin to read, a death the forge contradicts. @@ -963,8 +968,10 @@ class ProfileReport: def needs_review(self) -> int: counts = self.counts or {} - return sum(counts.get(status, 0) for status in REVIEW_STATUSES) + len( - self.orphaned + return ( + sum(counts.get(status, 0) for status in REVIEW_STATUSES) + + len(self.orphaned) + + len(self.stale_prose or ()) ) @@ -1964,6 +1971,8 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str: f"{count} {status.lower()}" for status, count in sorted(report.counts.items()) ) lines.append(f" {len(report.entries)} refs: {summary}") + for line in report.stale_prose or (): + lines.append(f" prose under an older pin: {line}") lines.append(f" {report.needs_review()} to review") return "\n".join(lines) @@ -2200,6 +2209,12 @@ def drift_score( """Rank profiles by how far they have drifted from their pin.""" counts = report.counts or {} score = sum(DRIFT_WEIGHTS.get(k, 0) * v for k, v in counts.items()) + # Prose a reader has to settle weighs as an ambiguous ref, prose + # --realign-prose moves by itself as a shifted one. + score += sum( + DRIFT_WEIGHTS["SHIFTED" if line.startswith("would recale") else "AMBIGUOUS"] + for line in report.stale_prose or () + ) if version is not None and not version.tag_matches_declared: if version.latest_release and version.latest_release != version.declared: score += 10 @@ -3490,14 +3505,21 @@ def _check_quota(count: int, offline: bool) -> None: raise SystemExit(1) -def _stale_prose_blocks(name: str, path: Path, args: argparse.Namespace) -> bool: +def _stale_prose_blocks( + name: str, path: Path, args: argparse.Namespace, + known: list[str] | None = None, +) -> bool: """Whether prose written under an older pin has to move first. Anchored from the current pin, a recale or a bump follows someone else's code (boom3 FileSystem.cpp:2125, written at b810234e, an unrelated OpenFileRead at 132dfddb). It moves first, from the pin it was written at. + `known` is the answer the report already holds for the same text. """ - stale = realign_prose(path, args.cache_dir, args.offline, dry_run=True) + stale = ( + known if known is not None + else realign_prose(path, args.cache_dir, args.offline, dry_run=True) + ) if stale: print( f"{name}: {len(stale)} prose citation(s) written under an older pin; " @@ -3507,6 +3529,28 @@ def _stale_prose_blocks(name: str, path: Path, args: argparse.Namespace) -> bool return bool(stale) +def _read_stale_prose( + name: str, path: Path, args: argparse.Namespace, failed: set[str] +) -> list[str]: + """What --realign-prose would do or refuse on this profile, as a report. + + Read on every pass, not only before a write: prose that blocks the pin + is review work whether or not this run tries to move it. + """ + try: + return realign_prose(path, args.cache_dir, args.offline, dry_run=True) + except upstream.RateLimitError: + raise + except upstream.NotCachedError as exc: + # Unread offline, as a report is: not clean, and not a failure. + return [f"not read offline: {exc}"] + except upstream.UpstreamError as exc: + # Not read is not clean: the bump would refuse on the same error. + print(f"{name}: prose not read, {exc}", file=sys.stderr) + failed.add(name) + return [f"not read: {exc}"] + + def _backfill( args: argparse.Namespace, name: str, profile: dict, report: ProfileReport, path: Path, @@ -3540,7 +3584,11 @@ def _apply_writes( if not args.dry_run: path.write_bytes(target.read_bytes()) return - if not (args.rebase_refs or args.bump_commit) or _stale_prose_blocks(name, path, args): + # A backfill has just written a pin: the report's reading predates it. + known = None if args.backfill_commits else report.stale_prose + if not (args.rebase_refs or args.bump_commit) or _stale_prose_blocks( + name, path, args, known + ): return with tempfile.TemporaryDirectory() as scratch: # Always work on a copy. A dry run reports what it left there, a real @@ -3706,6 +3754,7 @@ def _print_triage( ) -> None: """Profiles ranked by drift, worst first.""" ranked = [] + sampled = 0 for report in reports: profile = selected[report.name] version = None @@ -3727,20 +3776,31 @@ def _print_triage( refused = "commit history unreachable" print(f"{report.name}: {exc}", file=sys.stderr) if len(paths) > TRIAGE_PATH_SAMPLE and not refused: - print( - f"{report.name}: commit count sampled on " - f"{TRIAGE_PATH_SAMPLE} of {len(paths)} paths", - file=sys.stderr, - ) + sampled += 1 ranked.append((drift_score(report, version, commits), report, refused)) + if sampled: + # One line for the run: a notice per profile buried the ranking + # under 254 lines of stderr. + print( + f"commit count sampled on {TRIAGE_PATH_SAMPLE} cited paths for " + f"{sampled} profiles citing more", + file=sys.stderr, + ) + print(f"{'score':>5s} {'profile':30s} state (worst first)") + pending = 0 for score, report, refused in sorted(ranked, key=lambda item: -item[0]): state = report.skipped or f"{report.needs_review()} to review" + stale = len(report.stale_prose or ()) + if stale and not report.skipped: + state += f", {stale} prose under an older pin" unchecked = (report.counts or {}).get("UNCHECKED", 0) if unchecked and not report.skipped: state += f", {unchecked} unchecked" if refused: state += f", {refused}" + pending += bool(report.needs_review()) print(f"{score:5d} {report.name:30s} {state}") + print(f"{len(ranked)} profiles, {pending} with something to review") def _refuse_flag_combinations(args: argparse.Namespace) -> None: @@ -3895,6 +3955,8 @@ def main() -> None: name=name, entries=[], counts={}, skipped=f"upstream error: {exc}" ) failed.add(name) + if not report.skipped and profile_path.is_file(): + report.stale_prose = _read_stale_prose(name, profile_path, args, failed) if args.check_version and profile.get("version_source"): report.declared_version = str(profile.get("core_version") or "") or None report.page_version = page_version(profile, args.cache_dir, args.offline) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index aa852bd5..aa47269c 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -4002,6 +4002,25 @@ class TestOutputModesApplyTheirFlags(unittest.TestCase): code, _, _ = self._main("--strict") self.assertEqual(code, 0) + def test_strict_fails_on_prose_that_blocks_the_pin(self): + """blastem read "0 to review" while its prose refused every bump.""" + from unittest import mock + + with tempfile.TemporaryDirectory() as tmp: + for name in ("clean", "drift"): + (Path(tmp) / f"{name}.yml").write_text("emulator: x\n") + stale = {"clean": ["read again: notes: a.c:5 (pin moved)"], "drift": []} + with mock.patch.object( + profile_sync, "realign_prose", + side_effect=lambda path, *a, **k: stale[path.stem], + ): + code, out, _ = self._main( + "--changed-only", "--strict", "--emulators-dir", tmp, + ) + self.assertEqual(code, 1) + self.assertIn("clean", out) + self.assertIn("prose under an older pin", out) + def test_extras_are_refused_with_json(self): code, _, err = self._main("--json", "--check-version") self.assertEqual(code, 1) @@ -4504,5 +4523,52 @@ class ARoleIsReadPerPart(unittest.TestCase): ]) +class TheTriageCountsWhatBlocksThePin(unittest.TestCase): + """Prose written under an older pin refuses the recale and the bump, yet + the triage, --changed-only and --strict read "0 to review" on blastem, + xenia-edge and lindbergh-loader. And one stderr notice per profile put + 254 lines above an unlabelled ranking.""" + + def _report(self, name: str, paths: int, stale: list[str] | None = None): + entries = [ + EntryReport(f"f{i}", f"d/{i}.c:1", "ANCHORED", [ + PartResult(RefPart(f"d/{i}.c", 1, 1, "1"), "ANCHORED", + None, None, None, []) + ]) + for i in range(paths) + ] + return ProfileReport( + name=name, pin="p", head="h", entries=entries, + counts={"ANCHORED": paths}, stale_prose=stale, + ) + + def test_stale_prose_is_review_work(self): + report = self._report("b", 1, ["read again: notes: a.c:5 (pin moved)"]) + self.assertEqual(report.needs_review(), 1) + self.assertIn("prose under an older pin", format_report(report)) + self.assertEqual(report_to_dict(report)["needs_review"], 1) + + def test_the_ranking_is_labelled_and_the_sampling_said_once(self): + reports = [ + self._report("clean", 7), + self._report("stuck", 7, ["read again: notes: a.c:5 (pin moved)"]), + ] + selected = {r.name: {"source": "https://github.com/o/n"} for r in reports} + args = argparse.Namespace(cache_dir="unused", offline=True) + out, err = io.StringIO(), io.StringIO() + with mock.patch.object(profile_sync, "check_version", return_value=None), \ + mock.patch.object(profile_sync.upstream, "commits_touching", + return_value=0), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + profile_sync._print_triage(args, selected, reports) + lines = out.getvalue().splitlines() + self.assertIn("profile", lines[0]) + self.assertIn("stuck", lines[1]) + self.assertIn("1 to review, 1 prose under an older pin", lines[1]) + self.assertIn("0 to review", lines[2]) + self.assertEqual(lines[-1], "2 profiles, 1 with something to review") + self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue()) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/tools.md b/wiki/tools.md index 8164c5d4..fe9f1c36 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -415,7 +415,15 @@ not declare, so no revision can confirm it), `UNCHECKED` (a pin equal to HEAD is judged on self-consistency, and an entry with neither a hash nor a name gives that check nothing to look for). An entry carries the worst status of its parts. Only `CHANGED`, `GONE` and `AMBIGUOUS` count as -needing a re-read. A zero next to a row of `unchecked` is not a proof. +needing a re-read, with each prose citation `--realign-prose` would move or +refuse (written under an older pin, it holds the pin back). A zero next to +a row of `unchecked` is not a proof. + +A part opening with `source` or `upstream`, in a source_ref or in prose, +is read in that repository. The word belongs to the part, not to its path: +one profile can cite the same file under both. `--triage` prints a header, +ranks worst first and closes on the number of profiles with something to +review. A single cited line is often not distinctive, so the anchor widens by steps of ±3, ±6, ±12, ±25 and ±50 lines until it is unique. Three further From 94a230a06340bc90429d924d68907ba66b72cbdc Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:26:53 +0200 Subject: [PATCH 5/5] chore: frame the triage rows in a refusal test --- tests/test_audit_regressions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 2c500274..7adaea13 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3521,7 +3521,8 @@ class TriageSurvivesAForgeRefusal(unittest.TestCase): contextlib.redirect_stdout(out), \ contextlib.redirect_stderr(io.StringIO()): profile_sync._print_triage(args, selected, reports) - lines = out.getvalue().splitlines() + # A header row and a closing count frame the ranking. + lines = out.getvalue().splitlines()[1:-1] self.assertEqual(len(lines), 2) self.assertTrue(any("eden" in l and "unreachable" in l for l in lines))