From a0ecca002448268fdd9a3810489345b953cffaa3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:33:16 +0100 Subject: [PATCH] fix: collect abbreviated prose citations A location written without its file, (:507), reads in the file named last in the scalar, either cited or named as 'path:' before a list. A dotless file cited with its directory (tools/errorgen:20) is a citation too. devilutionx, corsixth, starship, plastic, eden-nightly, theforceengine, lightspark and higan_sfc gain 33 citations that bump and realign now follow. Two read that way pointed at another file and now name theirs: devilutionx assets.cpp:511-518, corsixth app.lua:1591. --- emulators/corsixth.yml | 2 +- emulators/devilutionx.yml | 2 +- scripts/sourceref.py | 126 ++++++++++++++++++++++++++----------- tests/test_profile_sync.py | 33 ++++++++++ 4 files changed, 123 insertions(+), 40 deletions(-) diff --git a/emulators/corsixth.yml b/emulators/corsixth.yml index 019755a9d..dc366dd4f 100644 --- a/emulators/corsixth.yml +++ b/emulators/corsixth.yml @@ -54,7 +54,7 @@ notes: | upper-case names the DOS release ships and the lookup normalises to. The sound/music path uses the game's XMI tracks when a SoundFont is - available (App:findSoundFont at :1591 probes GeneralUser-GS.sf2, + available (App:findSoundFont at CorsixTH/Lua/app.lua:1591 probes GeneralUser-GS.sf2, FluidR3_GM.sf2, FluidR3.sf3 and three Linux system paths), and the SoundFont is a user preference, not game data. diff --git a/emulators/devilutionx.yml b/emulators/devilutionx.yml index 0c7dfcb13..97825919c 100644 --- a/emulators/devilutionx.yml +++ b/emulators/devilutionx.yml @@ -54,7 +54,7 @@ notes: | the mods/hf file list under the BUILD_MOD_MPQS option (CMake/Mods.cmake:11-36, 61-71). The compiled binary expects devilutionx.mpq next to the executable on every - platform except Android, Apple and 3DS/Switch (:511-518), where its + platform except Android, Apple and 3DS/Switch (Source/engine/assets.cpp:511-518), where its contents are shipped loose. After loading the core archives the engine reads the asset ASSETS_VERSION and aborts with "Failed to load UI resources" unless its content is "1\n" (Source/init.cpp:54, 79-85, diff --git a/scripts/sourceref.py b/scripts/sourceref.py index c0bf3fcd1..ec980b6e2 100644 --- a/scripts/sourceref.py +++ b/scripts/sourceref.py @@ -220,10 +220,21 @@ PROSE_EXTENSIONLESS = ( "COPYING", "CHANGELOG", ) -PROSE_CITE_RE = re.compile( +# Any other dotless file is cited with its directory, which a constant or a +# hex address never carries: lightspark's tools/errorgen:20. +_PROSE_PATH = ( r"(?P[A-Za-z0-9_][\w./+-]*\.[A-Za-z]\w*" - r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r"))" - r":(?P\d+(?:-\d+)?)" + r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r")" + r"|(?:[\w.+-]+/)+[A-Za-z_][\w+-]*)" +) +PROSE_CITE_RE = re.compile(_PROSE_PATH + r":(?P\d+(?:-\d+)?)") +# A file named before its locations, `Source/engine/assets.cpp:` then a +# list, and a location that leaves the file out, `(:507)`, which reads in +# the file named last: devilutionx, corsixth and starship cite their load +# order this way. Neither was collected, so no bump or guard followed them. +_PROSE_FILE_NAMED_RE = re.compile(_PROSE_PATH + r":(?=\s|$)") +_PROSE_ABBREVIATED_RE = re.compile( + r"(?:(?<=[(\s])|^):(?P\d+(?:-\d+)?)(?![\w:-])(?!\.\d)" ) PROSE_LIST_ITEM_RE = re.compile(r"\d+[.)](?:\s|$)") PROSE_WRAPPED_RANGE_RE = re.compile(r":\d+(?:-\d+)?(?:[,)\s]|$)") @@ -344,6 +355,12 @@ def _prose_runs( joined line, so a writer can put each span back on its own line. """ lines = text.splitlines() + # The file an abbreviated location reads in, and the role it was named + # under, carried from line to line within the scalar. + named: tuple[str, str] | None = None + # Where the run wrapped from the line before ends in this one: its + # ranges are that run's, not abbreviated locations of their own. + carried = 0 for number, own in enumerate(lines): # A literal block keeps its line breaks, so a citation wrapped after # `file:` or after a range's comma lost the ranges on the next line @@ -369,46 +386,35 @@ def _prose_runs( elif following and line.endswith(","): joined_at = len(line) + 1 line += " " + following - consumed = 0 - for match in PROSE_CITE_RE.finditer(line): + consumed, carried = carried, 0 + events = sorted( + [(m.start(), 0, m) for m in PROSE_CITE_RE.finditer(line)] + + [(m.start(), 1, m) for m in _PROSE_FILE_NAMED_RE.finditer(line)] + + [(m.start(), 2, m) for m in _PROSE_ABBREVIATED_RE.finditer(line)], + key=lambda event: (event[0], event[1]), + ) + for _start, kind, match in events: if match.start() >= boundary: break if match.start() < consumed: continue - token_start = max(line.rfind(" ", 0, match.start()), - line.rfind("\t", 0, match.start())) + 1 - token = line[token_start:].split()[0] if line[token_start:] else "" - if "://" in token: + if kind == 1: + if "://" not in line[max(0, match.start() - 3):match.end()]: + named = (match.group("path"), "") continue - path = match.group("path") - path_start = match.start("path") - # A dotfile citation opens its token with the dot: `.gitlab-ci.yml` - # after a delimiter is the filename, while the dot of `boot.cfg` - # inside a word belongs to the word before it. - if ( - path_start > 0 - and line[path_start - 1] == "." - and (path_start == 1 or line[path_start - 2] in " \t(") - ): - path_start -= 1 - path = "." + path - # A path opening its line reads its role or project word at the - # end of the line before: genesis_plus_gx_expanded wrapped - # `(Batocera` and `package/batocera/...mk:7` one line apart. - lead = line[:path_start] - if number and not lead.strip(): - lead = lines[number - 1].rstrip() + " " - role = _prose_role(lead, len(lead)) - if not role and _prose_external(lead, len(lead), known): - # Kept with its project word, the source_ref convention: the - # report looks for the project inside the declared trees - # (mupen64plus-ae vendors `rice Config.cpp`) and calls the - # citation EXTERNAL only when none carries it. - before = lead.rstrip() - word = before[max(before.rfind(" "), before.rfind("\t"), - before.rfind("(")) + 1:] - path = f"{word} {path}" - path_span = (path_start, match.end("path")) + if kind == 2: + if named is None: + continue + path, role = named + path_start = match.start() + path_span = (path_start, path_start) + else: + path, role, path_start, path_span = _prose_path( + line, lines, number, match, known + ) + if path is None: + continue + named = (path, role) spans = [(match.start("range"), match.end("range"))] end = match.end() while True: @@ -429,6 +435,9 @@ def _prose_runs( spans.append((cont.start("range"), cont.end("range"))) end = cont.end() consumed = end + if end > boundary: + indent = len(lines[number + 1]) - len(lines[number + 1].lstrip()) + carried = indent + end - joined_at parts = [] for lo, hi in spans: rng = line[lo:hi] @@ -438,3 +447,44 @@ def _prose_runs( number, path_span, tuple(spans), tuple(parts), line[path_start:end], joined_at if end > boundary else 0, ) + + +def _prose_path( + line: str, lines: list[str], number: int, match: re.Match, + known: frozenset[str] | set[str], +) -> tuple[str | None, str, int, tuple[int, int]]: + """(path, role, start, span) of one written citation, None for a URL.""" + token_start = max(line.rfind(" ", 0, match.start()), + line.rfind("\t", 0, match.start())) + 1 + token = line[token_start:].split()[0] if line[token_start:] else "" + if "://" in token: + return None, "", 0, (0, 0) + path = match.group("path") + path_start = match.start("path") + # A dotfile citation opens its token with the dot: `.gitlab-ci.yml` + # after a delimiter is the filename, while the dot of `boot.cfg` + # inside a word belongs to the word before it. + if ( + path_start > 0 + and line[path_start - 1] == "." + and (path_start == 1 or line[path_start - 2] in " \t(") + ): + path_start -= 1 + path = "." + path + # A path opening its line reads its role or project word at the + # end of the line before: genesis_plus_gx_expanded wrapped + # `(Batocera` and `package/batocera/...mk:7` one line apart. + lead = line[:path_start] + if number and not lead.strip(): + lead = lines[number - 1].rstrip() + " " + role = _prose_role(lead, len(lead)) + if not role and _prose_external(lead, len(lead), known): + # Kept with its project word, the source_ref convention: the + # report looks for the project inside the declared trees + # (mupen64plus-ae vendors `rice Config.cpp`) and calls the + # citation EXTERNAL only when none carries it. + before = lead.rstrip() + word = before[max(before.rfind(" "), before.rfind("\t"), + before.rfind("(")) + 1:] + path = f"{word} {path}" + return path, role, path_start, (path_start, match.end("path")) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 2521591b2..8bee4ba3d 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -5919,6 +5919,39 @@ class ARoleIsReadPerPart(unittest.TestCase): ("src/a.cpp", "source"), ("src/a.cpp", "source"), ("b.c", "upstream"), ]) + def test_an_abbreviated_location_reads_in_the_file_named_last(self): + """devilutionx writes `Source/engine/assets.cpp:` then `(:507)` for + each loader, and its pin moved under five uncollected numbers.""" + from sourceref import _prose_runs + + text = ( + "Load order is defined in Source/engine/assets.cpp:\n" + " LoadCoreArchives() (:507) - devilutionx.mpq, fonts.mpq\n" + " LoadLanguageArchive() (:523) - .mpq\n" + "Startup (src/port/Engine.cpp:65-127) loads sf64.o2r (:72, :88-89),\n" + "falls back at :90-106. Wrapped (code/q_shared.h\n" + ":44,65) and lightspark's (tools/errorgen:20).\n" + ) + cited = [ + (p.path, p.start, p.end) for run in _prose_runs(text) for p in run[3] + ] + self.assertEqual(cited, [ + ("Source/engine/assets.cpp", 507, 507), + ("Source/engine/assets.cpp", 523, 523), + ("src/port/Engine.cpp", 65, 127), + ("src/port/Engine.cpp", 72, 72), + ("src/port/Engine.cpp", 88, 89), + ("src/port/Engine.cpp", 90, 106), + ("code/q_shared.h", 44, 44), + ("code/q_shared.h", 65, 65), + ("tools/errorgen", 20, 20), + ]) + + def test_an_abbreviated_location_needs_a_file_before_it(self): + from sourceref import _prose_runs + + self.assertEqual(list(_prose_runs("read at boot (:507), devilutionx.mpq")), []) + class TheTriageCountsWhatBlocksThePin(unittest.TestCase): """Prose written under an older pin refuses the recale and the bump, yet