"""The pages a newcomer reads stay short, plain and quick to act on.

The Start here pages ran 460 to 1,950 words, the first command of the
download page sat 2.8 phone screens down, and a sentence of 57 words listed
every platform's check. Each budget below is a number on the built page:
words before the first action, sentence and block length, words per page, and
a list of words a newcomer does not know or does not need.

Run ``python -m tests.test_site_copy --report docs`` to print the measures of
every page held here.
"""

from __future__ import annotations

import os
import re
import statistics
import sys
import unittest
from dataclasses import dataclass, field
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
DOCS = REPO_ROOT / "docs"

# Level 0: the Start here pages, with their own ceiling on words.
START_PAGES = {
    "index.md": 220,
    "which-pack.md": 400,
    "wiki/what-are-bios-files.md": 300,
    "wiki/first-install.md": 400,
    "wiki/troubleshooting.md": 300,
    "wiki/faq.md": 450,
    "wiki/glossary.md": 200,
    "wiki/getting-started.md": 120,
}
# Level 1: the guide part of each setup page, above its Reference heading.
SETUP_CEILING = 220
# The setup index is a card per setup, one sentence each.
SETUP_INDEX_CEILING = 300

BUDGETS = {
    #            longest sentence, median sentence, longest block, words before the first action
    "start": (20, 12, 35, 50),
    "setup": (25, 14, 40, 90),
}
HOME_BEFORE_ACTION = 40
LIST_ITEMS = {"start": 5, "setup": 6}
HEADING_WORDS = 8

# Words that say nothing, or talk about the page instead of the task. The
# marketing words are held on every page by test_prose_style.
FILLER = (
    "just", "simply", "easy", "easily", "easiest", "basically", "actually",
    "really", "very", "quite", "note that", "please note", "keep in mind",
    "of course", "obviously", "clearly", "in order to", "worth", "goes through",
    "covers", "explains", "says more", "says where it stands", "what follows",
    "this page", "these pages", "each section", "the last section",
    "the rest of this page", "further down", "the usual causes", "come up",
    "one path", "a few minutes", "is the answer", "the normal way", "not a fault",
    "does not pretend", "plainly", "for when you want", "for your own tools",
    "welcome",
    "ensure", "utilize", "facilitate", "via", "e.g.", "i.e.", "etc.", "vs.",
    "respectively", "deliberately",
)
# Words a newcomer meets only with their meaning in the same sentence. The
# value is the gloss that makes the word allowed there, None when no gloss
# does: such a word belongs one level down.
JARGON = {
    "checksum": None, "md5": None, "sha1": None, "sha-1": None, "sha-256": None,
    "crc32": None, "hash": r"fingerprint", "manifest": None, "bootstrap": None,
    "execution policy": None, "libretro": None, "standalone": None, "hle": None,
    "dump": None, "revision": None, "verified": None, "frontend": None,
    "system id": None, "upstream": None, "scrape": None, "yaml": None,
    "json": None, "csv": None, "sqlite": None, "repository": None,
    "release asset": None, "rom set": None, "casefold": None,
    "case-sensitive": None, "mount": None, "read-only": None, "proxy": None,
    "https_proxy": None, "sudo": r"administrator", "ssh": r"terminal over the network",
    "termux": r"terminal app", "core": r"small emulator|one emulator|its own emulator",
    "interoperability": None, "fair use": None, "dmca": None, "circumvent": None,
    "rerecording": None, "fpga cores": None, "de10-nano": None, "amlogic": None,
    "rk356x": None, "h700": None, "rk3399": None, "emulationstation": None,
    "extract": None, "extracted": None,
}
# Level 0 says ZIP and setup, a setup page names its own pack and platform.
START_ONLY = {"pack": None, "packs": None, "platform": None, "platforms": None}
# A word glossed in place: "X, a ...", "X (a ...", "called X".
GENERIC_GLOSS = r"(?:{w}\*{{0,2}}, an? |{w}\*{{0,2}} \(an? |called \*{{0,2}}{w})"

WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9'._/\\:%~-]*")
SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`\"(*\[])")


@dataclass
class Block:
    kind: str  # paragraph, item, term, definition, heading, table
    text: str
    collapsed: bool = False
    defines: str = ""
    hidden: bool = False  # in a tab other than the first: read instead of the first


@dataclass
class Page:
    blocks: list[Block] = field(default_factory=list)
    before_action: int | None = None
    list_runs: list[int] = field(default_factory=list)

    def prose(self) -> list[Block]:
        return [b for b in self.blocks if b.kind in ("paragraph", "item", "term", "definition")]

    def words(self) -> int:
        return sum(count_words(b.text) for b in self.blocks if b.kind != "table" and not b.hidden)

    def sentences(self) -> list[str]:
        return [s for b in self.prose() for s in split_sentences(b.text)]


def strip_inline(text: str) -> str:
    """Link text kept, targets, attribute lists, images and markup dropped."""
    text = re.sub(r"!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?", "", text)
    text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
    text = re.sub(r"\{\s*[#.:][^}]*\}", "", text)
    text = re.sub(r"<[^>]+>", "", text)
    return text.replace("**", "").strip()


def count_words(text: str) -> int:
    return len(WORD.findall(re.sub(r"`[^`]*`", "code", text)))


def split_sentences(text: str) -> list[str]:
    return [s for s in SENTENCE_END.split(text.strip()) if WORD.search(s)]


def guide_part(markdown: str) -> str:
    """A page up to its Reference heading, without front matter."""
    if markdown.startswith("---\n"):
        markdown = markdown.split("\n---\n", 1)[1]
    match = re.search(r"^## Reference\b", markdown, re.M)
    return markdown[: match.start()] if match else markdown


def parse(markdown: str) -> Page:
    """Blocks of prose a reader sees, code and HTML left out."""
    text = guide_part(markdown)
    text = re.sub(r"<script.*?</script>", "", text, flags=re.S)
    text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
    page = Page()
    seen_words = 0
    collapsed_indent: int | None = None
    tab_indent: int | None = None  # the tabs of a group a reader picks one of
    hidden_indent: int | None = None
    fence: str | None = None
    current: list[str] = []
    kind = "paragraph"
    run = 0

    def flush() -> None:
        nonlocal current, seen_words, kind
        if current:
            body = strip_inline(" ".join(current))
            if WORD.search(body):
                term = page.blocks[-1].text.lower() if kind == "definition" and page.blocks else ""
                block = Block(kind, body, collapsed_indent is not None, term,
                              hidden_indent is not None)
                page.blocks.append(block)
                if (page.before_action is None and not block.collapsed
                        and not block.hidden and kind not in ("table", "heading")):
                    seen_words += count_words(body)
        current = []
        if kind in ("item", "definition", "heading"):
            kind = "paragraph"

    def act() -> None:
        flush()
        if page.before_action is None:
            page.before_action = seen_words

    for raw in text.splitlines():
        line = raw.rstrip()
        stripped = line.strip()
        indent = len(line) - len(line.lstrip())
        if fence:
            if stripped.startswith(fence):
                fence = None
            continue
        if stripped and tab_indent is not None and indent <= tab_indent and not stripped.startswith("=== "):
            flush()
            tab_indent = hidden_indent = None
        if stripped.startswith(("```", "~~~")):
            fence = stripped[:3]
            if hidden_indent is None:
                act()
            continue
        if collapsed_indent is not None and stripped and indent <= collapsed_indent:
            flush()
            collapsed_indent = None
        if not stripped:
            flush()
            continue
        if re.match(r"(\?\?\?\+?|!!!) ", stripped):
            flush()
            if stripped.startswith("??? "):
                collapsed_indent = indent
            continue
        if stripped.startswith("=== "):
            flush()
            if tab_indent == indent:
                hidden_indent = indent
            else:
                tab_indent, hidden_indent = indent, None
            continue
        if stripped.startswith("<") or stripped.startswith("</"):
            flush()
            continue
        if ".md-button" in stripped:
            act()
            continue
        if stripped.startswith("#"):
            flush()
            if run:
                page.list_runs.append(run)
                run = 0
            kind = "heading"
            current = [stripped.lstrip("#").strip()]
            flush()
            kind = "paragraph"
            continue
        if stripped.startswith("|"):
            if kind != "table":
                flush()
                kind = "table"
            if not re.fullmatch(r"\|[\s:|-]+\|?", stripped):
                current.append(stripped.replace("|", " "))
            continue
        if kind == "table":
            flush()
            kind = "paragraph"
        item = re.match(r"(?:[-*+]|\d+\.)\s+(.*)", stripped)
        definition = re.match(r":\s+(.*)", stripped)
        if item:
            flush()
            kind = "item"
            run += 1
            current = [item.group(1)]
            continue
        if definition:
            flush()
            if page.blocks and page.blocks[-1].kind == "paragraph":
                page.blocks[-1].kind = "term"
            kind = "definition"
            current = [definition.group(1)]
            continue
        if kind == "item" and indent == 0:
            flush()
            kind = "paragraph"
        if kind == "paragraph" and run:
            page.list_runs.append(run)
            run = 0
        if not current and kind in ("item", "definition") and indent == 0:
            kind = "paragraph"
        current.append(stripped)
    flush()
    if run:
        page.list_runs.append(run)
    return page


def banned(block: Block, level: str) -> list[str]:
    """Filler, and jargon met without its meaning beside it."""
    found = []
    text = re.sub(r"`[^`]*`", "", block.text)
    if block.kind == "heading" and text.rstrip().endswith("."):
        return []  # a heading that quotes an installer message
    if block.kind == "term":
        return []
    for sentence in split_sentences(text) or [text]:
        lower = sentence.lower()
        for phrase in FILLER:
            if re.search(rf"(?<![\w-]){re.escape(phrase)}(?![\w-])", lower):
                found.append(phrase)
        terms = dict(JARGON)
        if level == "start":
            terms.update(START_ONLY)
        for term, gloss in terms.items():
            if not re.search(rf"(?<![\w-]){re.escape(term)}(?![\w-])", lower):
                continue
            if gloss and re.search(gloss, lower):
                continue
            if re.search(GENERIC_GLOSS.format(w=re.escape(term)), lower):
                continue
            if term in block.defines:
                continue
            found.append(term)
    return found


def measure(page: Page) -> dict:
    lengths = [count_words(s) for s in page.sentences()]
    return {
        "words": page.words(),
        "before_action": page.before_action,
        "max_block": max((count_words(b.text) for b in page.prose()), default=0),
        "median_sentence": statistics.median(lengths) if lengths else 0,
        "max_sentence": max(lengths, default=0),
    }


def held_pages(docs: Path) -> list[tuple[str, str, int]]:
    """(relative path, level, ceiling) for every page the budgets hold."""
    pages = [(name, "start", ceiling) for name, ceiling in START_PAGES.items()]
    pages += [
        (str(path.relative_to(docs)), "setup", SETUP_CEILING)
        for path in sorted((docs / "platforms").glob("*.md"))
        if path.name != "index.md"
        and "\n## Where the files go" in path.read_text(encoding="utf-8")
    ]
    pages.append(("platforms/index.md", "setup", SETUP_INDEX_CEILING))
    return [(name, level, ceiling) for name, level, ceiling in pages if (docs / name).is_file()]


def _docs() -> Path:
    if not (DOCS / "index.md").is_file():
        if os.environ.get("CI"):
            raise AssertionError("CI generates docs/ before the suite")
        raise unittest.SkipTest("docs/ not generated (python scripts/generate_site.py)")
    return DOCS


def _load(name: str) -> Page:
    return parse((_docs() / name).read_text(encoding="utf-8"))


class CopyBudgets(unittest.TestCase):
    def test_sentences_stay_short(self):
        for name, level, _ceiling in held_pages(_docs()):
            longest, median, _block, _before = BUDGETS[level]
            page = _load(name)
            with self.subTest(page=name):
                long = [s for s in page.sentences() if count_words(s) > longest]
                self.assertEqual(long, [], f"sentences over {longest} words")
                self.assertLessEqual(measure(page)["median_sentence"], median)

    def test_blocks_stay_short(self):
        for name, level, _ceiling in held_pages(_docs()):
            limit = BUDGETS[level][2]
            page = _load(name)
            with self.subTest(page=name):
                long = [b.text for b in page.prose() if count_words(b.text) > limit]
                self.assertEqual(long, [], f"blocks over {limit} words")

    def test_the_first_action_comes_early(self):
        for name, level, _ceiling in held_pages(_docs()):
            limit = HOME_BEFORE_ACTION if name == "index.md" else BUDGETS[level][3]
            page = _load(name)
            if page.before_action is None:
                continue  # a page of definitions has nothing to run
            with self.subTest(page=name):
                self.assertLessEqual(page.before_action, limit)

    def test_pages_stay_under_their_ceiling(self):
        for name, _level, ceiling in held_pages(_docs()):
            with self.subTest(page=name):
                self.assertLessEqual(_load(name).words(), ceiling)

    def test_lists_of_choices_stay_short(self):
        for name, level, _ceiling in held_pages(_docs()):
            with self.subTest(page=name):
                self.assertLessEqual(max(_load(name).list_runs, default=0), LIST_ITEMS[level])

    def test_headings_stay_short(self):
        for name, _level, _ceiling in held_pages(_docs()):
            page = _load(name)
            long = [
                b.text for b in page.blocks
                if b.kind == "heading" and not b.text.endswith(".")
                and count_words(re.sub(r"`[^`]*`", "", b.text)) > HEADING_WORDS
            ]
            with self.subTest(page=name):
                self.assertEqual(long, [])


class PlainWords(unittest.TestCase):
    def test_no_filler_and_no_unexplained_jargon(self):
        offenders = []
        for name, level, _ceiling in held_pages(_docs()):
            for block in _load(name).blocks:
                if block.kind == "table":
                    continue
                words = banned(block, level)
                if words:
                    offenders.append(f"{name}: {sorted(set(words))} in: {block.text[:90]}")
        self.assertEqual(offenders, [], "\n".join(offenders[:40]))


class TheMeasureItself(unittest.TestCase):
    """The parser decides every number above, so it is held on a sample."""

    SAMPLE = (
        "---\nlevel: start\n---\n\n# Title\n\nOne two three. Four five.\n\n"
        "- Item **one**.\n- Item [two](x.md).\n\n```sh\nnot counted at all\n```\n\n"
        "??? note \"Hidden\"\n\n    Collapsed words here.\n\n"
        "=== \"A\"\n\n    First tab.\n\n=== \"B\"\n\n    Second tab words.\n\n"
        "| A | B |\n|---|---|\n| cell | cell |\n\n## Reference: deep\n\nNot read.\n"
    )

    def test_blocks_and_counts(self):
        page = parse(self.SAMPLE)
        kinds = [b.kind for b in page.blocks]
        self.assertEqual(kinds, ["heading", "paragraph", "item", "item", "paragraph",
                                 "paragraph", "paragraph", "table"])
        self.assertEqual([b.hidden for b in page.blocks[5:7]], [False, True])
        self.assertEqual(page.words(), 1 + 5 + 2 + 2 + 3 + 2)
        self.assertEqual(page.before_action, 5 + 2 + 2)
        self.assertEqual(page.list_runs, [2])
        self.assertTrue(page.blocks[4].collapsed)
        self.assertEqual(page.sentences()[:2], ["One two three.", "Four five."])

    def test_a_gloss_allows_a_term(self):
        self.assertEqual(banned(Block("paragraph", "Install Termux, a terminal app."), "start"), [])
        self.assertEqual(banned(Block("paragraph", "Open Termux."), "start"), ["termux"])
        self.assertEqual(banned(Block("paragraph", "Run `sudo` first."), "start"), [])
        self.assertEqual(banned(Block("paragraph", "It is very easy."), "start"), ["easy", "very"])
        self.assertEqual(banned(Block("paragraph", "The RetroArch pack."), "setup"), [])


def report(docs: Path) -> str:
    rows = ["| Page | Words | Before action | Max block | Median / max sentence |",
            "|---|---:|---:|---:|---|"]
    for name, _level, _ceiling in held_pages(docs):
        m = measure(parse((docs / name).read_text(encoding="utf-8")))
        before = "none" if m["before_action"] is None else m["before_action"]
        rows.append(f"| {name} | {m['words']} | {before} | {m['max_block']} | "
                    f"{m['median_sentence']:g} / {m['max_sentence']} |")
    return "\n".join(rows)


if __name__ == "__main__":
    if len(sys.argv) == 3 and sys.argv[1] == "--report":
        print(report(Path(sys.argv[2])))
    else:
        unittest.main()