kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-30 15:47:20 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
70d761682c9c783688719da32c809951a0e773d3
libretro/tests/test_site_copy.py
T
Abdessamad Derraz 70d761682c docs: plain copy on the newcomer pages 
The Start here pages ran 460 to 2,100 words and the first command of
Get your files sat 254 words down. Each page now opens on its action,
with short sentences and one fact per block. The facts a newcomer does
not act on moved one level down: the legal analysis to wiki/legal.md,
the project terms to wiki/reference-terms.md, the annotated run and the
installer failures to the installer page, collection figures to a new
Explore index.

tests/test_site_copy.py holds the budgets on the generated pages, and
the testing guide states them with their reasons.

Four statements the repository contradicts are rewritten: prboom.wad
is compiled into PrBoom (emulators/prboom.yml), Batocera reports a
mismatch as UNTESTED (batocera-systems checkBios), 13 hle_fallback
entries are required, and the gaps headline splits missing firmware
from game data and unread entries.
2026-10-22 14:23:54 +02:00

443 lines
17 KiB
Python
Raw Blame History

"""The pages a newcomer reads stay short, plain and quick to act on.
The Start here pages ran 460 to 1,950 words, the first command of the
download page sat 2.8 phone screens down, and a sentence of 57 words listed
every platform's check. Each budget below is a number on the built page:
words before the first action, sentence and block length, words per page, and
a list of words a newcomer does not know or does not need.
Run ``python -m tests.test_site_copy --report docs`` to print the measures of
every page held here.
"""
from __future__ import annotations
import os
import re
import statistics
import sys
import unittest
from dataclasses import dataclass, field
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
DOCS = REPO_ROOT / "docs"
# Level 0: the Start here pages, with their own ceiling on words.
START_PAGES = {
"index.md": 220,
"which-pack.md": 400,
"wiki/what-are-bios-files.md": 300,
"wiki/first-install.md": 400,
"wiki/troubleshooting.md": 300,
"wiki/faq.md": 450,
"wiki/glossary.md": 200,
"wiki/getting-started.md": 120,
}
# Level 1: the guide part of each setup page, above its Reference heading.
SETUP_CEILING = 220
# The setup index is a card per setup, one sentence each.
SETUP_INDEX_CEILING = 300
BUDGETS = {
# longest sentence, median sentence, longest block, words before the first action
"start": (20, 12, 35, 50),
"setup": (25, 14, 40, 90),
}
HOME_BEFORE_ACTION = 40
LIST_ITEMS = {"start": 5, "setup": 6}
HEADING_WORDS = 8
# Words that say nothing, or talk about the page instead of the task.
FILLER = (
"just", "simply", "easy", "easily", "easiest", "basically", "actually",
"really", "very", "quite", "note that", "please note", "keep in mind",
"of course", "obviously", "clearly", "in order to", "worth", "goes through",
"covers", "explains", "says more", "says where it stands", "what follows",
"this page", "these pages", "each section", "the last section",
"the rest of this page", "further down", "the usual causes", "come up",
"one path", "a few minutes", "is the answer", "the normal way", "not a fault",
"does not pretend", "plainly", "for when you want", "for your own tools",
"welcome", "robust", "seamless", "comprehensive", "leverage", "streamline",
"ensure", "utilize", "facilitate", "via", "e.g.", "i.e.", "etc.", "vs.",
"respectively", "deliberately",
)
# Words a newcomer meets only with their meaning in the same sentence. The
# value is the gloss that makes the word allowed there, None when no gloss
# does: such a word belongs one level down.
JARGON = {
"checksum": None, "md5": None, "sha1": None, "sha-1": None, "sha-256": None,
"crc32": None, "hash": r"fingerprint", "manifest": None, "bootstrap": None,
"execution policy": None, "libretro": None, "standalone": None, "hle": None,
"dump": None, "revision": None, "verified": None, "frontend": None,
"system id": None, "upstream": None, "scrape": None, "yaml": None,
"json": None, "csv": None, "sqlite": None, "repository": None,
"release asset": None, "rom set": None, "casefold": None,
"case-sensitive": None, "mount": None, "read-only": None, "proxy": None,
"https_proxy": None, "sudo": r"administrator", "ssh": r"terminal over the network",
"termux": r"terminal app", "core": r"small emulator|one emulator|its own emulator",
"interoperability": None, "fair use": None, "dmca": None, "circumvent": None,
"rerecording": None, "fpga cores": None, "de10-nano": None, "amlogic": None,
"rk356x": None, "h700": None, "rk3399": None, "emulationstation": None,
"extract": None, "extracted": None,
}
# Level 0 says ZIP and setup, a setup page names its own pack and platform.
START_ONLY = {"pack": None, "packs": None, "platform": None, "platforms": None}
# A word glossed in place: "X, a ...", "X (a ...", "called X".
GENERIC_GLOSS = r"(?:{w}\*{{0,2}}, an? |{w}\*{{0,2}} \(an? |called \*{{0,2}}{w})"
WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9'._/\\:%~-]*")
SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`\"(*\[])")
@dataclass
class Block:
kind: str # paragraph, item, term, definition, heading, table
text: str
collapsed: bool = False
defines: str = ""
hidden: bool = False # in a tab other than the first: read instead of the first
@dataclass
class Page:
blocks: list[Block] = field(default_factory=list)
before_action: int | None = None
list_runs: list[int] = field(default_factory=list)
def prose(self) -> list[Block]:
return [b for b in self.blocks if b.kind in ("paragraph", "item", "term", "definition")]
def words(self) -> int:
return sum(count_words(b.text) for b in self.blocks if b.kind != "table" and not b.hidden)
def sentences(self) -> list[str]:
return [s for b in self.prose() for s in split_sentences(b.text)]
def strip_inline(text: str) -> str:
"""Link text kept, targets, attribute lists, images and markup dropped."""
text = re.sub(r"!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?", "", text)
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
text = re.sub(r"\{\s*[#.:][^}]*\}", "", text)
text = re.sub(r"<[^>]+>", "", text)
return text.replace("**", "").strip()
def count_words(text: str) -> int:
return len(WORD.findall(re.sub(r"`[^`]*`", "code", text)))
def split_sentences(text: str) -> list[str]:
return [s for s in SENTENCE_END.split(text.strip()) if WORD.search(s)]
def guide_part(markdown: str) -> str:
"""A page up to its Reference heading, without front matter."""
if markdown.startswith("---\n"):
markdown = markdown.split("\n---\n", 1)[1]
match = re.search(r"^## Reference\b", markdown, re.M)
return markdown[: match.start()] if match else markdown
def parse(markdown: str) -> Page:
"""Blocks of prose a reader sees, code and HTML left out."""
text = guide_part(markdown)
text = re.sub(r"<script.*?</script>", "", text, flags=re.S)
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
page = Page()
seen_words = 0
collapsed_indent: int | None = None
tab_indent: int | None = None # the tabs of a group a reader picks one of
hidden_indent: int | None = None
fence: str | None = None
current: list[str] = []
kind = "paragraph"
run = 0
def flush() -> None:
nonlocal current, seen_words, kind
if current:
body = strip_inline(" ".join(current))
if WORD.search(body):
term = page.blocks[-1].text.lower() if kind == "definition" and page.blocks else ""
block = Block(kind, body, collapsed_indent is not None, term,
hidden_indent is not None)
page.blocks.append(block)
if (page.before_action is None and not block.collapsed
and not block.hidden and kind not in ("table", "heading")):
seen_words += count_words(body)
current = []
if kind in ("item", "definition", "heading"):
kind = "paragraph"
def act() -> None:
flush()
if page.before_action is None:
page.before_action = seen_words
for raw in text.splitlines():
line = raw.rstrip()
stripped = line.strip()
indent = len(line) - len(line.lstrip())
if fence:
if stripped.startswith(fence):
fence = None
continue
if stripped and tab_indent is not None and indent <= tab_indent and not stripped.startswith("=== "):
flush()
tab_indent = hidden_indent = None
if stripped.startswith(("```", "~~~")):
fence = stripped[:3]
if hidden_indent is None:
act()
continue
if collapsed_indent is not None and stripped and indent <= collapsed_indent:
flush()
collapsed_indent = None
if not stripped:
flush()
continue
if re.match(r"(\?\?\?\+?|!!!) ", stripped):
flush()
if stripped.startswith("??? "):
collapsed_indent = indent
continue
if stripped.startswith("=== "):
flush()
if tab_indent == indent:
hidden_indent = indent
else:
tab_indent, hidden_indent = indent, None
continue
if stripped.startswith("<") or stripped.startswith("</"):
flush()
continue
if ".md-button" in stripped:
act()
continue
if stripped.startswith("#"):
flush()
if run:
page.list_runs.append(run)
run = 0
kind = "heading"
current = [stripped.lstrip("#").strip()]
flush()
kind = "paragraph"
continue
if stripped.startswith("|"):
if kind != "table":
flush()
kind = "table"
if not re.fullmatch(r"\|[\s:|-]+\|?", stripped):
current.append(stripped.replace("|", " "))
continue
if kind == "table":
flush()
kind = "paragraph"
item = re.match(r"(?:[-*+]|\d+\.)\s+(.*)", stripped)
definition = re.match(r":\s+(.*)", stripped)
if item:
flush()
kind = "item"
run += 1
current = [item.group(1)]
continue
if definition:
flush()
if page.blocks and page.blocks[-1].kind == "paragraph":
page.blocks[-1].kind = "term"
kind = "definition"
current = [definition.group(1)]
continue
if kind == "item" and indent == 0:
flush()
kind = "paragraph"
if kind == "paragraph" and run:
page.list_runs.append(run)
run = 0
if not current and kind in ("item", "definition") and indent == 0:
kind = "paragraph"
current.append(stripped)
flush()
if run:
page.list_runs.append(run)
return page
def banned(block: Block, level: str) -> list[str]:
"""Filler, and jargon met without its meaning beside it."""
found = []
text = re.sub(r"`[^`]*`", "", block.text)
if block.kind == "heading" and text.rstrip().endswith("."):
return [] # a heading that quotes an installer message
if block.kind == "term":
return []
for sentence in split_sentences(text) or [text]:
lower = sentence.lower()
for phrase in FILLER:
if re.search(rf"(?<![\w-]){re.escape(phrase)}(?![\w-])", lower):
found.append(phrase)
terms = dict(JARGON)
if level == "start":
terms.update(START_ONLY)
for term, gloss in terms.items():
if not re.search(rf"(?<![\w-]){re.escape(term)}(?![\w-])", lower):
continue
if gloss and re.search(gloss, lower):
continue
if re.search(GENERIC_GLOSS.format(w=re.escape(term)), lower):
continue
if term in block.defines:
continue
found.append(term)
return found
def measure(page: Page) -> dict:
lengths = [count_words(s) for s in page.sentences()]
return {
"words": page.words(),
"before_action": page.before_action,
"max_block": max((count_words(b.text) for b in page.prose()), default=0),
"median_sentence": statistics.median(lengths) if lengths else 0,
"max_sentence": max(lengths, default=0),
}
def held_pages(docs: Path) -> list[tuple[str, str, int]]:
"""(relative path, level, ceiling) for every page the budgets hold."""
pages = [(name, "start", ceiling) for name, ceiling in START_PAGES.items()]
pages += [
(str(path.relative_to(docs)), "setup", SETUP_CEILING)
for path in sorted((docs / "platforms").glob("*.md"))
if path.name != "index.md"
and "\n## Where the files go" in path.read_text(encoding="utf-8")
]
pages.append(("platforms/index.md", "setup", SETUP_INDEX_CEILING))
return [(name, level, ceiling) for name, level, ceiling in pages if (docs / name).is_file()]
def _docs() -> Path:
if not (DOCS / "index.md").is_file():
if os.environ.get("CI"):
raise AssertionError("CI generates docs/ before the suite")
raise unittest.SkipTest("docs/ not generated (python scripts/generate_site.py)")
return DOCS
def _load(name: str) -> Page:
return parse((_docs() / name).read_text(encoding="utf-8"))
class CopyBudgets(unittest.TestCase):
def test_sentences_stay_short(self):
for name, level, _ceiling in held_pages(_docs()):
longest, median, _block, _before = BUDGETS[level]
page = _load(name)
with self.subTest(page=name):
long = [s for s in page.sentences() if count_words(s) > longest]
self.assertEqual(long, [], f"sentences over {longest} words")
self.assertLessEqual(measure(page)["median_sentence"], median)
def test_blocks_stay_short(self):
for name, level, _ceiling in held_pages(_docs()):
limit = BUDGETS[level][2]
page = _load(name)
with self.subTest(page=name):
long = [b.text for b in page.prose() if count_words(b.text) > limit]
self.assertEqual(long, [], f"blocks over {limit} words")
def test_the_first_action_comes_early(self):
for name, level, _ceiling in held_pages(_docs()):
limit = HOME_BEFORE_ACTION if name == "index.md" else BUDGETS[level][3]
page = _load(name)
if page.before_action is None:
continue # a page of definitions has nothing to run
with self.subTest(page=name):
self.assertLessEqual(page.before_action, limit)
def test_pages_stay_under_their_ceiling(self):
for name, _level, ceiling in held_pages(_docs()):
with self.subTest(page=name):
self.assertLessEqual(_load(name).words(), ceiling)
def test_lists_of_choices_stay_short(self):
for name, level, _ceiling in held_pages(_docs()):
with self.subTest(page=name):
self.assertLessEqual(max(_load(name).list_runs, default=0), LIST_ITEMS[level])
def test_headings_stay_short(self):
for name, _level, _ceiling in held_pages(_docs()):
page = _load(name)
long = [
b.text for b in page.blocks
if b.kind == "heading" and not b.text.endswith(".")
and count_words(re.sub(r"`[^`]*`", "", b.text)) > HEADING_WORDS
]
with self.subTest(page=name):
self.assertEqual(long, [])
class PlainWords(unittest.TestCase):
def test_no_filler_and_no_unexplained_jargon(self):
offenders = []
for name, level, _ceiling in held_pages(_docs()):
for block in _load(name).blocks:
if block.kind == "table":
continue
words = banned(block, level)
if words:
offenders.append(f"{name}: {sorted(set(words))} in: {block.text[:90]}")
self.assertEqual(offenders, [], "\n".join(offenders[:40]))
class TheMeasureItself(unittest.TestCase):
"""The parser decides every number above, so it is held on a sample."""
SAMPLE = (
"---\nlevel: start\n---\n\n# Title\n\nOne two three. Four five.\n\n"
"- Item **one**.\n- Item [two](x.md).\n\n```sh\nnot counted at all\n```\n\n"
"??? note \"Hidden\"\n\n Collapsed words here.\n\n"
"=== \"A\"\n\n First tab.\n\n=== \"B\"\n\n Second tab words.\n\n"
"| A | B |\n|---|---|\n| cell | cell |\n\n## Reference: deep\n\nNot read.\n"
)
def test_blocks_and_counts(self):
page = parse(self.SAMPLE)
kinds = [b.kind for b in page.blocks]
self.assertEqual(kinds, ["heading", "paragraph", "item", "item", "paragraph",
"paragraph", "paragraph", "table"])
self.assertEqual([b.hidden for b in page.blocks[5:7]], [False, True])
self.assertEqual(page.words(), 1 + 5 + 2 + 2 + 3 + 2)
self.assertEqual(page.before_action, 5 + 2 + 2)
self.assertEqual(page.list_runs, [2])
self.assertTrue(page.blocks[4].collapsed)
self.assertEqual(page.sentences()[:2], ["One two three.", "Four five."])
def test_a_gloss_allows_a_term(self):
self.assertEqual(banned(Block("paragraph", "Install Termux, a terminal app."), "start"), [])
self.assertEqual(banned(Block("paragraph", "Open Termux."), "start"), ["termux"])
self.assertEqual(banned(Block("paragraph", "Run `sudo` first."), "start"), [])
self.assertEqual(banned(Block("paragraph", "It is very easy."), "start"), ["easy", "very"])
self.assertEqual(banned(Block("paragraph", "The RetroArch pack."), "setup"), [])
def report(docs: Path) -> str:
rows = ["| Page | Words | Before action | Max block | Median / max sentence |",
"|---|---:|---:|---:|---|"]
for name, _level, _ceiling in held_pages(docs):
m = measure(parse((docs / name).read_text(encoding="utf-8")))
before = "none" if m["before_action"] is None else m["before_action"]
rows.append(f"| {name} | {m['words']} | {before} | {m['max_block']} | "
f"{m['median_sentence']:g} / {m['max_sentence']} |")
return "\n".join(rows)
if __name__ == "__main__":
if len(sys.argv) == 3 and sys.argv[1] == "--report":
print(report(Path(sys.argv[2])))
else:
unittest.main()
Reference in new issue View Git Blame Copy Permalink