libretro/tests/test_prose_style.py

"""Prose in the repository is written the way its maintainer writes.

Em-dashes, clause-joining semicolons, unicode arrows, the ellipsis character,
box-drawing banners and check-mark emoji had spread through profiles, docs,
comments and command output. None of them is wrong, but together they gave the
project a generated look its maintainer did not want. This module names every
place one comes back.
"""

from __future__ import annotations

import io
import os
import re
import subprocess
import tokenize
import unittest
from pathlib import Path

import yaml

ROOT = Path(__file__).resolve().parent.parent

GLYPHS = re.compile(
    "[\u2013\u2014\u2026\u2190-\u21ff\u2500-\u257f\u2713-\u2718\u26a0\u2705\u274c]"
)
# A semicolon between two words of prose. Code quoted in prose sits in
# backticks, which are blanked before the search.
CLAUSE_SEMICOLON = re.compile(r"[A-Za-z0-9)\]\"']; [A-Za-z(]")
BACKTICKS = re.compile(r"`[^`\n]*`")
# "1 files", a count the generator printed without its singular.
ONE_PLURAL = re.compile(r"(?<![\w,./])1 (files|issues|entries|emulators|platforms|systems|cores)\b")
FENCED = re.compile(r"^```.*?^```", re.S | re.M)

SKIPPED_TREES = ("bios/", "data/", "recipes/", "provenance/", "install/")
TEXT_SUFFIXES = {
    ".py", ".sh", ".ps1", ".js", ".css", ".md", ".yml", ".yaml", ".html",
    ".txt", ".cfg", ".toml", "",
}
PROSE_KEYS = {
    "notes", "note", "description", "exclusion_note", "size_note",
    "region_note", "hle_note", "comment", "reason",
}


def _tracked() -> list[Path]:
    listed = subprocess.run(
        ["git", "ls-files"], cwd=ROOT, capture_output=True, text=True, check=True
    ).stdout.splitlines()
    return [
        ROOT / name
        for name in listed
        if not name.startswith(SKIPPED_TREES)
        and Path(name).suffix in TEXT_SUFFIXES
        and (ROOT / name).is_file()
    ]


def _semicolons(text: str) -> list[str]:
    return [
        line.strip()
        for line in BACKTICKS.sub("``", text).splitlines()
        if CLAUSE_SEMICOLON.search(line)
    ]


def _python_prose(source: str) -> str:
    """Comments and docstrings of a module, one per line."""
    parts: list[str] = []
    previous = tokenize.INDENT
    for token in tokenize.generate_tokens(io.StringIO(source).readline):
        if token.type == tokenize.COMMENT:
            parts.append(token.string)
        elif token.type == tokenize.STRING and previous in (
            tokenize.INDENT, tokenize.NEWLINE, tokenize.NL, tokenize.DEDENT,
        ):
            parts.append(token.string)
        if token.type not in (tokenize.COMMENT, tokenize.NL):
            previous = token.type
    return "\n".join(parts)


class _TagTolerantLoader(yaml.SafeLoader):
    """mkdocs.yml names Python callables with !!python/name tags."""


_TagTolerantLoader.add_multi_constructor(
    "tag:yaml.org,2002:python/", lambda loader, suffix, node: None
)


def _yaml_prose(text: str) -> str:
    """Prose scalars and comments of a YAML document."""
    parts = [
        line.split("#", 1)[1]
        for line in text.splitlines()
        if line.lstrip().startswith("#")
    ]

    def walk(node: object) -> None:
        if isinstance(node, dict):
            for key, value in node.items():
                if key in PROSE_KEYS and isinstance(value, str):
                    parts.append(value)
                else:
                    walk(value)
        elif isinstance(node, list):
            for item in node:
                walk(item)

    walk(yaml.load(text, Loader=_TagTolerantLoader))
    return "\n".join(parts)


# Words a heading keeps capitalized: names of platforms, systems and programs.
_HEADING_KEEP = set("""Batocera Recalbox Lakka Dolphin Redump Linux Windows Android Python Steam Deck
macOS Termux Flatpak Switch Raspberry Pi Mega Drive Genesis Saturn Dreamcast Amiga Atari Sega
Nintendo Sony Neo Geo Lynx Jaguar Commodore Apple Mac Spectrum Sinclair Amstrad MSX GameCube Wii
Game Boy Advance Famicom Disk System Super Master PlayStation Vita Xbox Arcade English French
Spanish I Lua Unity Qt Kodi Discord Pages Actions Docker Windows Mac Ubuntu Debian Arch
SteamOS Chihiro NAOMI Triforce Lindbergh Model PinMAME ScummVM Dinothawr Cemu Citra Yuzu Ryujinx
Mednafen Beetle Stella Hatari Caprice Vice Fuse Daphne Hypseus Singe Supermodel Flycast Redream
DuckStation SwanStation Opera Gearsystem Picodrive Snes9x Nestopia Mesen Gambatte SameBoy
mGBA melonDS DeSmuME Azahar Eden Sudachi Torzu Xemu Cxbx PCSX2 AetherSX2 NetherSX2 ARMSX2
RPCS3 Vita3K Zinc Raine FBNeo FinalBurn Neo MAME Clrmamepro RomVault""".split())


def _sentence_case(title: str) -> str:
    words = title.split(" ")
    out = []
    starts = True
    for w in words:
        core = w.strip("()[],:.")
        if starts:
            # The first word of a heading or of a clause after ":", "." or " - ".
            head, sep, tail = core.partition("-")
            if sep and re.fullmatch(r"[A-Z][a-z]+", tail) and tail not in _HEADING_KEEP:
                w = w.replace(core, f"{head}-{tail.lower()}", 1)
        elif re.fullmatch(r"[A-Z][a-z]+(-[A-Z]?[a-z]+)*", core) and core not in _HEADING_KEEP:
            w = w.replace(core, core.lower(), 1)
        out.append(w)
        starts = w.endswith((":", ".")) or w == "-" or re.fullmatch(r"\d+[a-z]?\.?", w) is not None
    return " ".join(out)


class NoGeneratedTypography(unittest.TestCase):
    def test_no_dash_arrow_ellipsis_or_banner_glyphs(self):
        offenders = []
        for path in _tracked():
            text = path.read_text(encoding="utf-8", errors="replace")
            for number, line in enumerate(text.splitlines(), 1):
                if GLYPHS.search(line):
                    offenders.append(f"{path.relative_to(ROOT)}:{number}: {line.strip()[:100]}")
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))


class NoMarketingVocabulary(unittest.TestCase):
    """The project bans these words in published text, and nothing checked
    it: "streamlined interface" reached the site's platform table."""

    WORDS = re.compile(
        r"\b(comprehensive|robust|seamless(ly)?|leverag\w*|streamlin\w*|delv(e|ing))\b",
        re.IGNORECASE,
    )

    def test_no_tracked_text_uses_them(self):
        offenders = []
        for path in _tracked():
            if path.name == "test_prose_style.py":
                continue
            text = path.read_text(encoding="utf-8", errors="replace")
            for number, line in enumerate(text.splitlines(), 1):
                if self.WORDS.search(line):
                    offenders.append(f"{path.relative_to(ROOT)}:{number}: {line.strip()[:100]}")
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))


class NoToolingNames(unittest.TestCase):
    """Nothing committed names the private notes or the agent tooling the
    maintainer works with. Two test docstrings did, and history keeps them."""

    # The names the repository keeps out of version control through its
    # private exclude list, plus the assistant itself. `claude\w*` covers
    # claudedocs/.
    NAMES = re.compile(
        r"(?i)\b(claude\w*|anthropic)\b|\b(prompt|agents|audit)\.md\b"
        r"|\.(claude|codex|cursor|continue|copilot|gemini|windsurf|serena|playwright-mcp)/"
        r"|\.aider"
    )

    def test_the_pattern_covers_every_private_name(self):
        """The pattern caught CLAUDE.md and missed AGENTS.md, AUDIT.md and
        claudedocs/, which the same rule keeps out of the repository."""
        for text in (
            "see CLAUDE.md", "rules live in AGENTS.md", "AUDIT.md angle G",
            "notes in claudedocs/launchers", "prompt.md", "under .claude/hooks",
            "a .cursor/ rule", "the .serena/ cache", ".aider.conf.yml",
        ):
            with self.subTest(text=text):
                self.assertRegex(text, self.NAMES)
        for text in ("When Superpowers Collide", "audit_regressions", "an agent of change"):
            with self.subTest(text=text):
                self.assertNotRegex(text, self.NAMES)

    def test_no_tracked_file_names_them(self):
        offenders = []
        for path in _tracked():
            if path.name == "test_prose_style.py":
                continue
            text = path.read_text(encoding="utf-8", errors="replace")
            for number, line in enumerate(text.splitlines(), 1):
                if self.NAMES.search(line):
                    offenders.append(f"{path.relative_to(ROOT)}:{number}")
        self.assertEqual(offenders, [], "\n".join(offenders[:20]))


class ProfilesStateFactsAboutOtherProjects(unittest.TestCase):
    """A profile says what another project's file declares and what the code
    does, never a verdict on that file. Ten notes and comments in ten profiles
    called .info entries "phantom", "misleading" or "incorrectly" written, and
    the site rendered them on the emulator pages."""

    VERDICTS = re.compile(
        r"\b(phantoms?|misleading|bogus|typos?|has errors|incorrectly "
        r"(?:says|lists|declares|states|claims|names|describes))\b",
        re.IGNORECASE,
    )

    def test_profile_prose_and_comments(self):
        offenders = [
            f"{path.relative_to(ROOT)}: {line.strip()[:100]}"
            for path in sorted((ROOT / "emulators").glob("*.yml"))
            for line in _yaml_prose(path.read_text(encoding="utf-8")).splitlines()
            if self.VERDICTS.search(line)
        ]
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))


class NoClauseSemicolons(unittest.TestCase):
    def _check(self, files: list[Path], extract) -> None:
        offenders = []
        for path in files:
            text = path.read_text(encoding="utf-8")
            for line in _semicolons(extract(text)):
                offenders.append(f"{path.relative_to(ROOT)}: {line[:100]}")
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))

    def test_markdown(self):
        files = [p for p in _tracked() if p.suffix == ".md"]
        self._check(files, lambda text: FENCED.sub("", text))

    def test_markdown_headings_are_headings(self):
        """Troubleshooting carried 21 bold lines in place of headings: no
        anchor to link from an issue, no entry in the page's contents."""
        pseudo = re.compile(r"^\*\*[^*]+:\*\*$", re.M)
        offenders = [
            f"{p.relative_to(ROOT)}: {m.group(0)}"
            for p in _tracked() if p.suffix == ".md"
            for m in pseudo.finditer(FENCED.sub("", p.read_text(encoding="utf-8")))
        ]
        self.assertEqual(offenders, [], "\n".join(offenders[:20]))

    def test_wiki_headings_use_sentence_case(self):
        """The gaps page and two wiki pages titled in Title Case ("What Each
        Pack Contains") beside sentence-case pages ("Quick install")."""
        offenders = []
        for page in sorted((ROOT / "wiki").glob("*.md")):
            text = FENCED.sub("", page.read_text(encoding="utf-8"))
            for title in re.findall(r"(?m)^#{1,4} (.+)$", text):
                if _sentence_case(title) != title:
                    offenders.append(f"{page.name}: {title}")
        self.assertEqual(offenders, [], "\n".join(offenders))

    def test_yaml_prose_and_comments(self):
        files = [p for p in _tracked() if p.suffix in (".yml", ".yaml")]
        self._check(files, _yaml_prose)

    def test_python_comments_and_docstrings(self):
        files = [p for p in _tracked() if p.suffix == ".py"]
        self._check(files, _python_prose)


    def test_python_prose_has_no_lone_dash_asides(self):
        """Removing " -- " asides left 72 comments reading "shape -a size or
        a size range -is all": half a dash glued to the next word. A command
        line flag quoted in prose (MAME's -listxml) is not an aside, and a
        quotation is left alone."""
        lone = re.compile(r"[A-Za-z0-9,)] -(?!listxml\b|lr\b|sa\b)[a-z]{2,}")
        offenders = [
            f"{path.relative_to(ROOT)}: {line.strip()[:100]}"
            for path in _tracked() if path.suffix == ".py"
            for line in re.sub(
                r'`[^`]*`|"[^"\n]*\n?[^"\n]*"',
                "",
                _python_prose(path.read_text(encoding="utf-8")).replace('"""', ""),
            ).splitlines()
            if lone.search(line) and not re.search(r"\b(python|curl|unittest|sh)\b .* -", line)
        ]
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))

    def test_schema_descriptions(self):
        offenders = []
        for path in sorted((ROOT / "schemas").glob("*.json")):
            text = path.read_text(encoding="utf-8")
            for line in _semicolons(text) + GLYPHS.findall(text):
                offenders.append(f"{path.relative_to(ROOT)}: {line[:100]}")
        self.assertEqual(offenders, [], "\n".join(offenders))


class GeneratedSiteProse(unittest.TestCase):
    """The site pages carry strings no tracked file shows verbatim."""

    def test_generated_pages(self):
        docs = ROOT / "docs"
        pages = [
            p for p in docs.rglob("*.md")
            if "superpowers" not in p.parts and "plans" not in p.parts
            and "specs" not in p.parts
        ] if docs.is_dir() else []
        if not pages:
            if os.environ.get("CI"):
                self.fail("CI generates docs/ before the suite")
            self.skipTest("docs/ not generated (python scripts/generate_site.py)")
        offenders = []
        for page in pages:
            text = re.sub(r"<script.*?</script>", "", page.read_text(encoding="utf-8"), flags=re.S)
            text = FENCED.sub("", text)
            for number, line in enumerate(text.splitlines(), 1):
                if GLYPHS.search(line) or _semicolons(line) or ONE_PLURAL.search(line):
                    offenders.append(f"{page.relative_to(ROOT)}:{number}: {line.strip()[:100]}")
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))


if __name__ == "__main__":
    unittest.main()