mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
Abdessamad Derraz
c5ba17d3dd
The new answers are shortened and the safety list, shown in full on the home page, is linked from each setup page instead of repeated. The setup ceiling stays at 220.
456 lines
18 KiB
Python
456 lines
18 KiB
Python
"""The pages a newcomer reads stay short, plain and quick to act on.
|
|
|
|
The Start here pages ran 460 to 1,950 words, the first command of the
|
|
download page sat 2.8 phone screens down, and a sentence of 57 words listed
|
|
every platform's check. Each budget below is a number on the built page:
|
|
words before the first action, sentence and block length, words per page, and
|
|
a list of words a newcomer does not know or does not need.
|
|
|
|
Run ``python -m tests.test_site_copy --report docs`` to print the measures of
|
|
every page held here.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import statistics
|
|
import sys
|
|
import unittest
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
from tests import sitepages # noqa: E402
|
|
|
|
DOCS = sitepages.CONTENT
|
|
|
|
# Level 0: the Start here pages, with their own ceiling on words.
|
|
START_PAGES = {
|
|
"index.md": 220,
|
|
"which-pack.md": 400,
|
|
"wiki/what-are-bios-files.md": 300,
|
|
"wiki/first-install.md": 400,
|
|
"wiki/troubleshooting.md": 300,
|
|
"wiki/faq.md": 450,
|
|
"wiki/glossary.md": 200,
|
|
}
|
|
# Level 1: the guide part of each setup page, above its Reference heading.
|
|
SETUP_CEILING = 220
|
|
# The setup index is a card per setup, one sentence each.
|
|
SETUP_INDEX_CEILING = 300
|
|
|
|
BUDGETS = {
|
|
# longest sentence, median sentence, longest block, words before the first action
|
|
"start": (20, 12, 35, 50),
|
|
"setup": (25, 14, 40, 90),
|
|
}
|
|
HOME_BEFORE_ACTION = 40
|
|
LIST_ITEMS = {"start": 5, "setup": 6}
|
|
HEADING_WORDS = 8
|
|
|
|
# Words that say nothing, or talk about the page instead of the task. The
|
|
# marketing words are held on every page by test_prose_style.
|
|
FILLER = (
|
|
"just", "simply", "easy", "easily", "easiest", "basically", "actually",
|
|
"really", "very", "quite", "note that", "please note", "keep in mind",
|
|
"of course", "obviously", "clearly", "in order to", "worth", "goes through",
|
|
"covers", "explains", "says more", "says where it stands", "what follows",
|
|
"this page", "these pages", "each section", "the last section",
|
|
"the rest of this page", "further down", "the usual causes", "come up",
|
|
"one path", "a few minutes", "is the answer", "the normal way", "not a fault",
|
|
"does not pretend", "plainly", "for when you want", "for your own tools",
|
|
"welcome",
|
|
"ensure", "utilize", "facilitate", "via", "e.g.", "i.e.", "etc.", "vs.",
|
|
"respectively", "deliberately",
|
|
)
|
|
# Words a newcomer meets only with their meaning in the same sentence. The
|
|
# value is the gloss that makes the word allowed there, None when no gloss
|
|
# does: such a word belongs one level down.
|
|
JARGON = {
|
|
"checksum": None, "md5": None, "sha1": None, "sha-1": None, "sha-256": None,
|
|
"crc32": None, "hash": r"fingerprint", "manifest": None, "bootstrap": None,
|
|
"execution policy": None, "libretro": None, "standalone": None, "hle": None,
|
|
"dump": None, "revision": None, "verified": None, "frontend": None,
|
|
"system id": None, "upstream": None, "scrape": None, "yaml": None,
|
|
"json": None, "csv": None, "sqlite": None, "repository": None,
|
|
"release asset": None, "rom set": None, "casefold": None,
|
|
"case-sensitive": None, "mount": None, "read-only": None, "proxy": None,
|
|
"https_proxy": None, "sudo": r"administrator", "ssh": r"terminal over the network",
|
|
"termux": r"terminal app", "core": r"small emulator|one emulator|its own emulator",
|
|
"interoperability": None, "fair use": None, "dmca": None, "circumvent": None,
|
|
"rerecording": None, "fpga cores": None, "de10-nano": None, "amlogic": None,
|
|
"rk356x": None, "h700": None, "rk3399": None, "emulationstation": None,
|
|
"extract": None, "extracted": None,
|
|
}
|
|
# Level 0 says ZIP and setup, a setup page names its own pack and platform.
|
|
START_ONLY = {"pack": None, "packs": None, "platform": None, "platforms": None}
|
|
# A word glossed in place: "X, a ...", "X (a ...", "called X".
|
|
GENERIC_GLOSS = r"(?:{w}\*{{0,2}}, an? |{w}\*{{0,2}} \(an? |called \*{{0,2}}{w})"
|
|
|
|
WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9'._/\\:%~-]*")
|
|
SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`\"(*\[])")
|
|
|
|
|
|
@dataclass
|
|
class Block:
|
|
kind: str # paragraph, item, term, definition, heading, table
|
|
text: str
|
|
collapsed: bool = False
|
|
defines: str = ""
|
|
hidden: bool = False # in a tab other than the first: read instead of the first
|
|
|
|
|
|
@dataclass
|
|
class Page:
|
|
blocks: list[Block] = field(default_factory=list)
|
|
before_action: int | None = None
|
|
list_runs: list[int] = field(default_factory=list)
|
|
|
|
def prose(self) -> list[Block]:
|
|
return [b for b in self.blocks if b.kind in ("paragraph", "item", "term", "definition")]
|
|
|
|
def words(self) -> int:
|
|
return sum(count_words(b.text) for b in self.blocks if b.kind != "table" and not b.hidden)
|
|
|
|
def sentences(self) -> list[str]:
|
|
return [s for b in self.prose() for s in split_sentences(b.text)]
|
|
|
|
|
|
def strip_inline(text: str) -> str:
|
|
"""Link text kept, targets, attribute lists, images and markup dropped."""
|
|
text = re.sub(r"!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?", "", text)
|
|
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
|
|
text = re.sub(r"\{\s*[#.:][^}]*\}", "", text)
|
|
text = re.sub(r"<[^>]+>", "", text)
|
|
return text.replace("**", "").strip()
|
|
|
|
|
|
def count_words(text: str) -> int:
|
|
return len(WORD.findall(re.sub(r"`[^`]*`", "code", text)))
|
|
|
|
|
|
def split_sentences(text: str) -> list[str]:
|
|
return [s for s in SENTENCE_END.split(text.strip()) if WORD.search(s)]
|
|
|
|
|
|
def guide_part(markdown: str) -> str:
|
|
"""A page up to its Reference heading, without front matter."""
|
|
if markdown.startswith("---\n"):
|
|
markdown = markdown.split("\n---\n", 1)[1]
|
|
match = re.search(r"^## Reference\b", markdown, re.M)
|
|
return markdown[: match.start()] if match else markdown
|
|
|
|
|
|
def parse(markdown: str) -> Page:
|
|
"""Blocks of prose a reader sees, code and HTML left out.
|
|
|
|
A collapsed block (<details>) counts, but not before the first action:
|
|
the reader opens it on purpose. Of a group of tabs the first is read and
|
|
the others only stand in for it.
|
|
"""
|
|
text = guide_part(markdown)
|
|
text = re.sub(r"<script.*?</script>", "", text, flags=re.S)
|
|
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
|
|
page = Page()
|
|
seen_words = 0
|
|
details = 0 # depth of collapsed blocks
|
|
tab = 0 # the tab of a group being read, 0 outside a group
|
|
fence: str | None = None
|
|
current: list[str] = []
|
|
kind = "paragraph"
|
|
run = 0
|
|
|
|
def flush() -> None:
|
|
nonlocal current, seen_words, kind
|
|
if current:
|
|
body = strip_inline(" ".join(current))
|
|
if WORD.search(body):
|
|
term = page.blocks[-1].text.lower() if kind == "definition" and page.blocks else ""
|
|
block = Block(kind, body, details > 0, term, tab > 1)
|
|
page.blocks.append(block)
|
|
if (page.before_action is None and not block.collapsed
|
|
and not block.hidden and kind not in ("table", "heading")):
|
|
seen_words += count_words(body)
|
|
current = []
|
|
if kind in ("item", "definition", "heading"):
|
|
kind = "paragraph"
|
|
|
|
def act() -> None:
|
|
flush()
|
|
if page.before_action is None:
|
|
page.before_action = seen_words
|
|
|
|
for raw in text.splitlines():
|
|
line = raw.rstrip()
|
|
stripped = line.strip()
|
|
indent = len(line) - len(line.lstrip())
|
|
if fence:
|
|
if stripped.startswith(fence):
|
|
fence = None
|
|
continue
|
|
if stripped.startswith(("```", "~~~")):
|
|
fence = stripped[:3]
|
|
if tab < 2:
|
|
act()
|
|
continue
|
|
if not stripped:
|
|
flush()
|
|
continue
|
|
if stripped.startswith("import ") and stripped.endswith(";"):
|
|
continue
|
|
if stripped.startswith(":::"):
|
|
flush()
|
|
continue
|
|
if stripped.startswith("<details"):
|
|
flush()
|
|
details += 1
|
|
continue
|
|
if stripped.startswith("</details>"):
|
|
flush()
|
|
details = max(0, details - 1)
|
|
continue
|
|
if stripped.startswith("<Tabs"):
|
|
flush()
|
|
tab = 0
|
|
continue
|
|
if stripped.startswith("<TabItem"):
|
|
flush()
|
|
tab += 1
|
|
continue
|
|
if stripped.startswith("</Tabs>"):
|
|
flush()
|
|
tab = 0
|
|
continue
|
|
if "rb-button" in stripped or "rb-tile" in stripped or stripped.startswith("<FilePicker"):
|
|
act()
|
|
continue
|
|
if re.match(r"</?(?!abbr\b|a\b|code\b|strong\b|em\b)[A-Za-z]", stripped):
|
|
flush()
|
|
continue
|
|
if stripped.startswith("#"):
|
|
flush()
|
|
if run:
|
|
page.list_runs.append(run)
|
|
run = 0
|
|
kind = "heading"
|
|
current = [re.sub(r"\s*\{[^}]*\}\s*$", "", stripped.lstrip("#").strip())]
|
|
flush()
|
|
kind = "paragraph"
|
|
continue
|
|
if stripped.startswith("|"):
|
|
if kind != "table":
|
|
flush()
|
|
kind = "table"
|
|
if not re.fullmatch(r"\|[\s:|-]+\|?", stripped):
|
|
current.append(stripped.replace("|", " "))
|
|
continue
|
|
if kind == "table":
|
|
flush()
|
|
kind = "paragraph"
|
|
item = re.match(r"(?:[-*+]|\d+\.)\s+(.*)", stripped)
|
|
definition = re.match(r":\s+(.*)", stripped)
|
|
if item:
|
|
flush()
|
|
kind = "item"
|
|
run += 1
|
|
current = [item.group(1)]
|
|
continue
|
|
if definition:
|
|
flush()
|
|
if page.blocks and page.blocks[-1].kind == "paragraph":
|
|
page.blocks[-1].kind = "term"
|
|
kind = "definition"
|
|
current = [definition.group(1)]
|
|
continue
|
|
if kind == "item" and indent == 0:
|
|
flush()
|
|
kind = "paragraph"
|
|
if kind == "paragraph" and run:
|
|
page.list_runs.append(run)
|
|
run = 0
|
|
if not current and kind in ("item", "definition") and indent == 0:
|
|
kind = "paragraph"
|
|
current.append(stripped)
|
|
flush()
|
|
if run:
|
|
page.list_runs.append(run)
|
|
return page
|
|
|
|
|
|
def banned(block: Block, level: str) -> list[str]:
|
|
"""Filler, and jargon met without its meaning beside it."""
|
|
found = []
|
|
text = re.sub(r"`[^`]*`", "", block.text)
|
|
if block.kind == "heading" and text.rstrip().endswith("."):
|
|
return [] # a heading that quotes an installer message
|
|
if block.kind == "term":
|
|
return []
|
|
for sentence in split_sentences(text) or [text]:
|
|
lower = sentence.lower()
|
|
for phrase in FILLER:
|
|
if re.search(rf"(?<![\w-]){re.escape(phrase)}(?![\w-])", lower):
|
|
found.append(phrase)
|
|
terms = dict(JARGON)
|
|
if level == "start":
|
|
terms.update(START_ONLY)
|
|
for term, gloss in terms.items():
|
|
if not re.search(rf"(?<![\w-]){re.escape(term)}(?![\w-])", lower):
|
|
continue
|
|
if gloss and re.search(gloss, lower):
|
|
continue
|
|
if re.search(GENERIC_GLOSS.format(w=re.escape(term)), lower):
|
|
continue
|
|
if term in block.defines:
|
|
continue
|
|
found.append(term)
|
|
return found
|
|
|
|
|
|
def measure(page: Page) -> dict:
|
|
lengths = [count_words(s) for s in page.sentences()]
|
|
return {
|
|
"words": page.words(),
|
|
"before_action": page.before_action,
|
|
"max_block": max((count_words(b.text) for b in page.prose()), default=0),
|
|
"median_sentence": statistics.median(lengths) if lengths else 0,
|
|
"max_sentence": max(lengths, default=0),
|
|
}
|
|
|
|
|
|
def held_pages(docs: Path) -> list[tuple[str, str, int]]:
|
|
"""(relative path, level, ceiling) for every page the budgets hold."""
|
|
pages = [(name, "start", ceiling) for name, ceiling in START_PAGES.items()]
|
|
pages += [
|
|
(sitepages.name(path), "setup", SETUP_CEILING)
|
|
for path in sorted((docs / "platforms").glob("*.md*"))
|
|
if path.stem != "index"
|
|
and "\n## Where the files go" in path.read_text(encoding="utf-8")
|
|
]
|
|
pages.append(("platforms/index.md", "setup", SETUP_INDEX_CEILING))
|
|
return [(name, level, ceiling) for name, level, ceiling in pages
|
|
if sitepages.page(name).is_file()]
|
|
|
|
|
|
def _docs() -> Path:
|
|
return sitepages.require()
|
|
|
|
|
|
def _load(name: str) -> Page:
|
|
_docs()
|
|
return parse(sitepages.page(name).read_text(encoding="utf-8"))
|
|
|
|
|
|
class CopyBudgets(unittest.TestCase):
|
|
def test_sentences_stay_short(self):
|
|
for name, level, _ceiling in held_pages(_docs()):
|
|
longest, median, _block, _before = BUDGETS[level]
|
|
page = _load(name)
|
|
with self.subTest(page=name):
|
|
long = [s for s in page.sentences() if count_words(s) > longest]
|
|
self.assertEqual(long, [], f"sentences over {longest} words")
|
|
self.assertLessEqual(measure(page)["median_sentence"], median)
|
|
|
|
def test_blocks_stay_short(self):
|
|
for name, level, _ceiling in held_pages(_docs()):
|
|
limit = BUDGETS[level][2]
|
|
page = _load(name)
|
|
with self.subTest(page=name):
|
|
long = [b.text for b in page.prose() if count_words(b.text) > limit]
|
|
self.assertEqual(long, [], f"blocks over {limit} words")
|
|
|
|
def test_the_first_action_comes_early(self):
|
|
for name, level, _ceiling in held_pages(_docs()):
|
|
limit = HOME_BEFORE_ACTION if name == "index.md" else BUDGETS[level][3]
|
|
page = _load(name)
|
|
if page.before_action is None:
|
|
continue # a page of definitions has nothing to run
|
|
with self.subTest(page=name):
|
|
self.assertLessEqual(page.before_action, limit)
|
|
|
|
def test_pages_stay_under_their_ceiling(self):
|
|
for name, _level, ceiling in held_pages(_docs()):
|
|
with self.subTest(page=name):
|
|
self.assertLessEqual(_load(name).words(), ceiling)
|
|
|
|
def test_lists_of_choices_stay_short(self):
|
|
for name, level, _ceiling in held_pages(_docs()):
|
|
with self.subTest(page=name):
|
|
self.assertLessEqual(max(_load(name).list_runs, default=0), LIST_ITEMS[level])
|
|
|
|
def test_headings_stay_short(self):
|
|
for name, _level, _ceiling in held_pages(_docs()):
|
|
page = _load(name)
|
|
long = [
|
|
b.text for b in page.blocks
|
|
if b.kind == "heading" and not b.text.endswith(".")
|
|
and count_words(re.sub(r"`[^`]*`", "", b.text)) > HEADING_WORDS
|
|
]
|
|
with self.subTest(page=name):
|
|
self.assertEqual(long, [])
|
|
|
|
|
|
class PlainWords(unittest.TestCase):
|
|
def test_no_filler_and_no_unexplained_jargon(self):
|
|
offenders = []
|
|
for name, level, _ceiling in held_pages(_docs()):
|
|
for block in _load(name).blocks:
|
|
if block.kind == "table":
|
|
continue
|
|
words = banned(block, level)
|
|
if words:
|
|
offenders.append(f"{name}: {sorted(set(words))} in: {block.text[:90]}")
|
|
self.assertEqual(offenders, [], "\n".join(offenders[:40]))
|
|
|
|
|
|
class TheMeasureItself(unittest.TestCase):
|
|
"""The parser decides every number above, so it is held on a sample."""
|
|
|
|
SAMPLE = (
|
|
"---\nlevel: start\n---\n\n# Title\n\nOne two three. Four five.\n\n"
|
|
"- Item **one**.\n- Item [two](x.md).\n\n```sh\nnot counted at all\n```\n\n"
|
|
'<details class="rb-details rb-details-note">\n<summary>Hidden</summary>\n\n'
|
|
"Collapsed words here.\n\n</details>\n\n"
|
|
'<Tabs syncKey="os">\n<TabItem label="A">\n\nFirst tab.\n\n</TabItem>\n'
|
|
'<TabItem label="B">\n\nSecond tab words.\n\n</TabItem>\n</Tabs>\n\n'
|
|
"| A | B |\n|---|---|\n| cell | cell |\n\n## Reference: deep\n\nNot read.\n"
|
|
)
|
|
|
|
def test_blocks_and_counts(self):
|
|
page = parse(self.SAMPLE)
|
|
kinds = [b.kind for b in page.blocks]
|
|
self.assertEqual(kinds, ["heading", "paragraph", "item", "item", "paragraph",
|
|
"paragraph", "paragraph", "table"])
|
|
self.assertEqual([b.hidden for b in page.blocks[5:7]], [False, True])
|
|
self.assertEqual(page.words(), 1 + 5 + 2 + 2 + 3 + 2)
|
|
self.assertEqual(page.before_action, 5 + 2 + 2)
|
|
self.assertEqual(page.list_runs, [2])
|
|
self.assertTrue(page.blocks[4].collapsed)
|
|
self.assertEqual(page.sentences()[:2], ["One two three.", "Four five."])
|
|
|
|
def test_a_gloss_allows_a_term(self):
|
|
self.assertEqual(banned(Block("paragraph", "Install Termux, a terminal app."), "start"), [])
|
|
self.assertEqual(banned(Block("paragraph", "Open Termux."), "start"), ["termux"])
|
|
self.assertEqual(banned(Block("paragraph", "Run `sudo` first."), "start"), [])
|
|
self.assertEqual(banned(Block("paragraph", "It is very easy."), "start"), ["easy", "very"])
|
|
self.assertEqual(banned(Block("paragraph", "The RetroArch pack."), "setup"), [])
|
|
|
|
|
|
def report(docs: Path) -> str:
|
|
rows = ["| Page | Words | Before action | Max block | Median / max sentence |",
|
|
"|---|---:|---:|---:|---|"]
|
|
for name, _level, _ceiling in held_pages(docs):
|
|
m = measure(parse(sitepages.page(name).read_text(encoding="utf-8")))
|
|
before = "none" if m["before_action"] is None else m["before_action"]
|
|
rows.append(f"| {name} | {m['words']} | {before} | {m['max_block']} | "
|
|
f"{m['median_sentence']:g} / {m['max_sentence']} |")
|
|
return "\n".join(rows)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) == 3 and sys.argv[1] == "--report":
|
|
print(report(Path(sys.argv[2])))
|
|
else:
|
|
unittest.main()
|