From c229daac3aa5453bcc889636315e634b78c6ad77 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:38:47 +0200 Subject: [PATCH] refactor: move the source_ref grammar out of sync The site imported two parsing functions from profile_sync, so every change to the comparison tool redeployed an unchanged site. The grammar now lives in sourceref.py. --- .github/workflows/deploy-site.yml | 2 +- scripts/profile_sync.py | 159 +++--------------------------- scripts/sitesources.py | 4 +- scripts/sourceref.py | 156 +++++++++++++++++++++++++++++ tests/test_workflow_paths.py | 6 ++ 5 files changed, 179 insertions(+), 148 deletions(-) create mode 100644 scripts/sourceref.py diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 368c9590..fab077ac 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -26,7 +26,6 @@ on: - "scripts/packpaths.py" - "scripts/packreadme.py" - "scripts/packresolve.py" - - "scripts/profile_sync.py" - "scripts/provenance_report.py" - "scripts/refresh_data_dirs.py" - "scripts/region.py" @@ -38,6 +37,7 @@ on: - "scripts/sitesources.py" - "scripts/sitewrite.py" - "scripts/slots.py" + - "scripts/sourceref.py" - "scripts/split_pack.py" - "scripts/truth.py" - "scripts/upstream.py" diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index dd9587d2..0c981221 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -32,6 +32,20 @@ sys.path.insert(0, os.path.dirname(__file__)) import upstream from common import load_emulator_profiles, suggest_names from hashing import parse_hash_list +from sourceref import ( # noqa: F401 (re-exported for callers and tests) + _ANNOTATION_RE, + _BARE_RANGE_RE, + _LOCATED_RE, + _REF_RE, + _SPLIT_RE, + RefPart, + _trim_prose, + parse_source_ref, + ref_prose_text, + source_ref_values, + split_outside_parentheses, + split_source_ref, +) from upstream import CompareResult, find_renamed DEFAULT_CACHE = upstream.CACHE_DIR @@ -59,25 +73,7 @@ MAX_MATCH_LINES = 60000 PIN = "pin" HEAD = "head" -_REF_RE = re.compile(r"^(?P[^:]+?)(?::(?P\d+)(?:-(?P\d+))?)?$") -_BARE_RANGE_RE = re.compile(r"^:?(\d+(?:-\d+)?)$") HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx", ".inc", ".inl") -_SPLIT_RE = re.compile(r"[,;]") -_ANNOTATION_RE = re.compile(r"\s*\([^)]*\)?") -_LOCATED_RE = re.compile(r"^[^\s:]+:\d+(?:-\d+)?$") - - -@dataclass(frozen=True) -class RefPart: - path: str - start: int | None - end: int | None - raw: str = "" - - @property - def last(self) -> int | None: - """Last cited line: the end of a range, or the single line.""" - return self.end or self.start @dataclass(frozen=True) @@ -113,133 +109,6 @@ class RepoView: field: str -def parse_source_ref(ref: str) -> tuple[str, int | None, int | None]: - """Split 'path/file.cpp:123-129' into (path, start, end).""" - m = _REF_RE.match(ref.strip()) - if not m: - return ref.strip(), None, None - start = int(m.group("start")) if m.group("start") else None - end = int(m.group("end")) if m.group("end") else start - return m.group("path"), start, end - - -def source_ref_values(value: str | dict | None) -> list[tuple[str, str]]: - """Flatten a source_ref into (label, refs) pairs. - - Most entries hold a plain string. `ymir` keys its refs by mode instead, - since its standalone and libretro builds read different files. - """ - if isinstance(value, dict): - return [(str(k), str(v)) for k, v in value.items() if v] - return [("", str(value))] if value else [] - - -def split_source_ref(ref: str) -> list[RefPart]: - """A source_ref may carry several references. - - Separators are the comma and the semicolon. A part reduced to a line or a - line range continues the previous part's file: `geo.c:234-243, 273-285` - cites two ranges of the same file, a form used by 430 parts across 69 - profiles. Parenthetical annotations are prose and are dropped: 285 entries - across 50 profiles carry them. - """ - parts: list[RefPart] = [] - for raw in split_outside_parentheses(str(ref or "")): - chunk = _trim_prose(_ANNOTATION_RE.sub("", raw).strip()) - if not chunk: - continue - bare = _BARE_RANGE_RE.match(chunk) - if bare and parts: - start, _, end = bare.group(1).partition("-") - parts.append( - RefPart(parts[-1].path, int(start), int(end or start), raw) - ) - continue - path, start_line, end_line = parse_source_ref(chunk) - parts.append(RefPart(path, start_line, end_line, raw)) - return parts - - -def ref_prose_text(ref: str) -> str: - """The text of a source_ref its parts do not read, positions kept. - - split_source_ref keeps the located head of each chunk and drops the - rest: annotations, and prose after the head. A citation written there - (`(MACHINE_IS_SPECTRUM_128_P2 covers Pentagon, src/cpu.h:545)`, or the - second half of `pce.cpp:477-483 + huc.cpp:284-304`) was never anchored. - Every character a part reads is blanked, so the prose scanner finds - the rest at the offsets it has in the scalar. - """ - masked = list(ref) - depth = 0 - outside: list[int] = [] - - def settle() -> None: - text = "".join(ref[i] for i in outside) - head = text.strip().split(" ", 1)[0] if text.strip() else "" - located = bool(head) and bool( - _LOCATED_RE.match(head) or _BARE_RANGE_RE.match(head) - ) - blank = outside - if located: - first = next(k for k, i in enumerate(outside) if not ref[i].isspace()) - blank = outside[first:first + len(head)] - for i in blank: - masked[i] = " " - outside.clear() - - for index, char in enumerate(ref): - if char == "(": - depth += 1 - if depth == 0: - if char in ",;": - settle() - else: - outside.append(index) - if char == ")": - depth = max(0, depth - 1) - settle() - return "".join(masked) - - -def split_outside_parentheses(text: str) -> list[str]: - """Split on commas and semicolons, ignoring those inside an annotation. - - Each chunk keeps its annotation, so a rewrite can put the prose back. - """ - chunks: list[str] = [] - current: list[str] = [] - depth = 0 - for char in text: - if char == "(": - depth += 1 - elif char == ")": - depth = max(0, depth - 1) - if char in ",;" and depth == 0: - chunks.append("".join(current)) - current = [] - else: - current.append(char) - chunks.append("".join(current)) - return [c for c in (chunk.strip() for chunk in chunks) if c] - - -def _trim_prose(chunk: str) -> str: - """Drop a trailing comment written without parentheses. - - `libretro.cpp:1520-1521 candidates_a1200` cites a location followed by a - note. The leading token is kept only when it already reads as a reference, - so `munt ROMInfo.cpp`, which names an external project, stays intact and - is reported rather than silently truncated to `munt`. - """ - if " " not in chunk: - return chunk - head = chunk.split(" ", 1)[0] - if _LOCATED_RE.match(head) or _BARE_RANGE_RE.match(head): - return head - return chunk - - HASH_FIELDS = ("sha1", "md5", "crc32", "sha256", "known_hash_adler32") diff --git a/scripts/sitesources.py b/scripts/sitesources.py index f776c1b9..436877c6 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -7,8 +7,8 @@ from __future__ import annotations from siterender import _by_mode import re -from profile_sync import source_ref_values -from profile_sync import split_source_ref +from sourceref import source_ref_values +from sourceref import split_source_ref import upstream import urllib.request def _repo_pins(profile: dict, field: str, label: str) -> list[tuple[str, str, str]]: diff --git a/scripts/sourceref.py b/scripts/sourceref.py new file mode 100644 index 00000000..53c63750 --- /dev/null +++ b/scripts/sourceref.py @@ -0,0 +1,156 @@ +"""The grammar of a profile's source_ref: where a reference points. + +Read by the site to link citations and by profile_sync to check them. Kept +apart so that building the site does not depend on the comparison tool. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_REF_RE = re.compile(r"^(?P[^:]+?)(?::(?P\d+)(?:-(?P\d+))?)?$") +_BARE_RANGE_RE = re.compile(r"^:?(\d+(?:-\d+)?)$") +_SPLIT_RE = re.compile(r"[,;]") +_ANNOTATION_RE = re.compile(r"\s*\([^)]*\)?") +_LOCATED_RE = re.compile(r"^[^\s:]+:\d+(?:-\d+)?$") + + +@dataclass(frozen=True) +class RefPart: + path: str + start: int | None + end: int | None + raw: str = "" + + @property + def last(self) -> int | None: + """Last cited line: the end of a range, or the single line.""" + return self.end or self.start + + +def parse_source_ref(ref: str) -> tuple[str, int | None, int | None]: + """Split 'path/file.cpp:123-129' into (path, start, end).""" + m = _REF_RE.match(ref.strip()) + if not m: + return ref.strip(), None, None + start = int(m.group("start")) if m.group("start") else None + end = int(m.group("end")) if m.group("end") else start + return m.group("path"), start, end + + +def source_ref_values(value: str | dict | None) -> list[tuple[str, str]]: + """Flatten a source_ref into (label, refs) pairs. + + Most entries hold a plain string. `ymir` keys its refs by mode instead, + since its standalone and libretro builds read different files. + """ + if isinstance(value, dict): + return [(str(k), str(v)) for k, v in value.items() if v] + return [("", str(value))] if value else [] + + +def split_source_ref(ref: str) -> list[RefPart]: + """A source_ref may carry several references. + + Separators are the comma and the semicolon. A part reduced to a line or a + line range continues the previous part's file: `geo.c:234-243, 273-285` + cites two ranges of the same file, a form used by 430 parts across 69 + profiles. Parenthetical annotations are prose and are dropped: 285 entries + across 50 profiles carry them. + """ + parts: list[RefPart] = [] + for raw in split_outside_parentheses(str(ref or "")): + chunk = _trim_prose(_ANNOTATION_RE.sub("", raw).strip()) + if not chunk: + continue + bare = _BARE_RANGE_RE.match(chunk) + if bare and parts: + start, _, end = bare.group(1).partition("-") + parts.append( + RefPart(parts[-1].path, int(start), int(end or start), raw) + ) + continue + path, start_line, end_line = parse_source_ref(chunk) + parts.append(RefPart(path, start_line, end_line, raw)) + return parts + + +def ref_prose_text(ref: str) -> str: + """The text of a source_ref its parts do not read, positions kept. + + split_source_ref keeps the located head of each chunk and drops the + rest: annotations, and prose after the head. A citation written there + (`(MACHINE_IS_SPECTRUM_128_P2 covers Pentagon, src/cpu.h:545)`, or the + second half of `pce.cpp:477-483 + huc.cpp:284-304`) was never anchored. + Every character a part reads is blanked, so the prose scanner finds + the rest at the offsets it has in the scalar. + """ + masked = list(ref) + depth = 0 + outside: list[int] = [] + + def settle() -> None: + text = "".join(ref[i] for i in outside) + head = text.strip().split(" ", 1)[0] if text.strip() else "" + located = bool(head) and bool( + _LOCATED_RE.match(head) or _BARE_RANGE_RE.match(head) + ) + blank = outside + if located: + first = next(k for k, i in enumerate(outside) if not ref[i].isspace()) + blank = outside[first:first + len(head)] + for i in blank: + masked[i] = " " + outside.clear() + + for index, char in enumerate(ref): + if char == "(": + depth += 1 + if depth == 0: + if char in ",;": + settle() + else: + outside.append(index) + if char == ")": + depth = max(0, depth - 1) + settle() + return "".join(masked) + + +def split_outside_parentheses(text: str) -> list[str]: + """Split on commas and semicolons, ignoring those inside an annotation. + + Each chunk keeps its annotation, so a rewrite can put the prose back. + """ + chunks: list[str] = [] + current: list[str] = [] + depth = 0 + for char in text: + if char == "(": + depth += 1 + elif char == ")": + depth = max(0, depth - 1) + if char in ",;" and depth == 0: + chunks.append("".join(current)) + current = [] + else: + current.append(char) + chunks.append("".join(current)) + return [c for c in (chunk.strip() for chunk in chunks) if c] + + +def _trim_prose(chunk: str) -> str: + """Drop a trailing comment written without parentheses. + + `libretro.cpp:1520-1521 candidates_a1200` cites a location followed by a + note. The leading token is kept only when it already reads as a reference, + so `munt ROMInfo.cpp`, which names an external project, stays intact and + is reported rather than silently truncated to `munt`. + """ + if " " not in chunk: + return chunk + head = chunk.split(" ", 1)[0] + if _LOCATED_RE.match(head) or _BARE_RANGE_RE.match(head): + return head + return chunk diff --git a/tests/test_workflow_paths.py b/tests/test_workflow_paths.py index 3c66765b..2b302876 100644 --- a/tests/test_workflow_paths.py +++ b/tests/test_workflow_paths.py @@ -101,6 +101,12 @@ class DeploySiteTriggers(unittest.TestCase): self.assertEqual(sorted(listed - closure(entry_points)), []) self.assertNotIn("tests/**", patterns) + def test_the_site_reads_the_ref_grammar_not_the_checker(self): + """sitesources imported two parsing functions from profile_sync, so + every change to the comparison tool rebuilt an unchanged site.""" + self.assertNotIn("scripts/profile_sync.py", closure(["generate_site.py"])) + self.assertIn("scripts/sourceref.py", closure(["generate_site.py"])) + class DocumentedSteps(unittest.TestCase): def test_the_release_page_names_every_installed_package(self):