kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-23 20:29:06 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
06e37b36f700b6ffe908a544007cd7f700afc6ce
libretro/tests/test_site_copy.py
T
Abdessamad Derraz 009413944e docs: open the deep pages on their question 
Each reference and contributor page now says what it answers and gives
a real case before its rules: scph5501.bin for the check modes and the
data model, DuckStation's entry for profiles, RomM for a platform. The
verification example is the RetroBat d64_1.rom case the gaps page lists
instead of a scph5501.bin case the data cannot produce.

Emulator pages open on what the emulator reads, from the profile, and
keep their technical notes folded. System pages say what a card holds.
The data model shows a real database entry and links the one
resolution chain. Gaps tables name what they count ("Discrepancy",
"Where the files come from"), and the core inventory is titled Cores
by platform so cross-reference keeps one meaning.
2026-10-22 14:31:57 +02:00

444 lines
17 KiB
Python
Raw Blame History

"""The pages a newcomer reads stay short, plain and quick to act on.
The Start here pages ran 460 to 1,950 words, the first command of the
download page sat 2.8 phone screens down, and a sentence of 57 words listed
every platform's check. Each budget below is a number on the built page:
words before the first action, sentence and block length, words per page, and
a list of words a newcomer does not know or does not need.
Run ``python -m tests.test_site_copy --report docs`` to print the measures of
every page held here.
"""
from __future__ import annotations
import os
import re
import statistics
import sys
import unittest
from dataclasses import dataclass, field
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
DOCS = REPO_ROOT / "docs"
# Level 0: the Start here pages, with their own ceiling on words.
START_PAGES = {
"index.md": 220,
"which-pack.md": 400,
"wiki/what-are-bios-files.md": 300,
"wiki/first-install.md": 400,
"wiki/troubleshooting.md": 300,
"wiki/faq.md": 450,
"wiki/glossary.md": 200,
"wiki/getting-started.md": 120,
}
# Level 1: the guide part of each setup page, above its Reference heading.
SETUP_CEILING = 220
# The setup index is a card per setup, one sentence each.
SETUP_INDEX_CEILING = 300
BUDGETS = {
# longest sentence, median sentence, longest block, words before the first action
"start": (20, 12, 35, 50),
"setup": (25, 14, 40, 90),
}
HOME_BEFORE_ACTION = 40
LIST_ITEMS = {"start": 5, "setup": 6}
HEADING_WORDS = 8
# Words that say nothing, or talk about the page instead of the task. The
# marketing words are held on every page by test_prose_style.
FILLER = (
"just", "simply", "easy", "easily", "easiest", "basically", "actually",
"really", "very", "quite", "note that", "please note", "keep in mind",
"of course", "obviously", "clearly", "in order to", "worth", "goes through",
"covers", "explains", "says more", "says where it stands", "what follows",
"this page", "these pages", "each section", "the last section",
"the rest of this page", "further down", "the usual causes", "come up",
"one path", "a few minutes", "is the answer", "the normal way", "not a fault",
"does not pretend", "plainly", "for when you want", "for your own tools",
"welcome",
"ensure", "utilize", "facilitate", "via", "e.g.", "i.e.", "etc.", "vs.",
"respectively", "deliberately",
)
# Words a newcomer meets only with their meaning in the same sentence. The
# value is the gloss that makes the word allowed there, None when no gloss
# does: such a word belongs one level down.
JARGON = {
"checksum": None, "md5": None, "sha1": None, "sha-1": None, "sha-256": None,
"crc32": None, "hash": r"fingerprint", "manifest": None, "bootstrap": None,
"execution policy": None, "libretro": None, "standalone": None, "hle": None,
"dump": None, "revision": None, "verified": None, "frontend": None,
"system id": None, "upstream": None, "scrape": None, "yaml": None,
"json": None, "csv": None, "sqlite": None, "repository": None,
"release asset": None, "rom set": None, "casefold": None,
"case-sensitive": None, "mount": None, "read-only": None, "proxy": None,
"https_proxy": None, "sudo": r"administrator", "ssh": r"terminal over the network",
"termux": r"terminal app", "core": r"small emulator|one emulator|its own emulator",
"interoperability": None, "fair use": None, "dmca": None, "circumvent": None,
"rerecording": None, "fpga cores": None, "de10-nano": None, "amlogic": None,
"rk356x": None, "h700": None, "rk3399": None, "emulationstation": None,
"extract": None, "extracted": None,
}
# Level 0 says ZIP and setup, a setup page names its own pack and platform.
START_ONLY = {"pack": None, "packs": None, "platform": None, "platforms": None}
# A word glossed in place: "X, a ...", "X (a ...", "called X".
GENERIC_GLOSS = r"(?:{w}\*{{0,2}}, an? |{w}\*{{0,2}} \(an? |called \*{{0,2}}{w})"
WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9'._/\\:%~-]*")
SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`\"(*\[])")
@dataclass
class Block:
kind: str # paragraph, item, term, definition, heading, table
text: str
collapsed: bool = False
defines: str = ""
hidden: bool = False # in a tab other than the first: read instead of the first
@dataclass
class Page:
blocks: list[Block] = field(default_factory=list)
before_action: int | None = None
list_runs: list[int] = field(default_factory=list)
def prose(self) -> list[Block]:
return [b for b in self.blocks if b.kind in ("paragraph", "item", "term", "definition")]
def words(self) -> int:
return sum(count_words(b.text) for b in self.blocks if b.kind != "table" and not b.hidden)
def sentences(self) -> list[str]:
return [s for b in self.prose() for s in split_sentences(b.text)]
def strip_inline(text: str) -> str:
"""Link text kept, targets, attribute lists, images and markup dropped."""
text = re.sub(r"!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?", "", text)
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
text = re.sub(r"\{\s*[#.:][^}]*\}", "", text)
text = re.sub(r"<[^>]+>", "", text)
return text.replace("**", "").strip()
def count_words(text: str) -> int:
return len(WORD.findall(re.sub(r"`[^`]*`", "code", text)))
def split_sentences(text: str) -> list[str]:
return [s for s in SENTENCE_END.split(text.strip()) if WORD.search(s)]
def guide_part(markdown: str) -> str:
"""A page up to its Reference heading, without front matter."""
if markdown.startswith("---\n"):
markdown = markdown.split("\n---\n", 1)[1]
match = re.search(r"^## Reference\b", markdown, re.M)
return markdown[: match.start()] if match else markdown
def parse(markdown: str) -> Page:
"""Blocks of prose a reader sees, code and HTML left out."""
text = guide_part(markdown)
text = re.sub(r"<script.*?</script>", "", text, flags=re.S)
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
page = Page()
seen_words = 0
collapsed_indent: int | None = None
tab_indent: int | None = None # the tabs of a group a reader picks one of
hidden_indent: int | None = None
fence: str | None = None
current: list[str] = []
kind = "paragraph"
run = 0
def flush() -> None:
nonlocal current, seen_words, kind
if current:
body = strip_inline(" ".join(current))
if WORD.search(body):
term = page.blocks[-1].text.lower() if kind == "definition" and page.blocks else ""
block = Block(kind, body, collapsed_indent is not None, term,
hidden_indent is not None)
page.blocks.append(block)
if (page.before_action is None and not block.collapsed
and not block.hidden and kind not in ("table", "heading")):
seen_words += count_words(body)
current = []
if kind in ("item", "definition", "heading"):
kind = "paragraph"
def act() -> None:
flush()
if page.before_action is None:
page.before_action = seen_words
for raw in text.splitlines():
line = raw.rstrip()
stripped = line.strip()
indent = len(line) - len(line.lstrip())
if fence:
if stripped.startswith(fence):
fence = None
continue
if stripped and tab_indent is not None and indent <= tab_indent and not stripped.startswith("=== "):
flush()
tab_indent = hidden_indent = None
if stripped.startswith(("```", "~~~")):
fence = stripped[:3]
if hidden_indent is None:
act()
continue
if collapsed_indent is not None and stripped and indent <= collapsed_indent:
flush()
collapsed_indent = None
if not stripped:
flush()
continue
if re.match(r"(\?\?\?\+?|!!!) ", stripped):
flush()
if stripped.startswith("??? "):
collapsed_indent = indent
continue
if stripped.startswith("=== "):
flush()
if tab_indent == indent:
hidden_indent = indent
else:
tab_indent, hidden_indent = indent, None
continue
if stripped.startswith("<") or stripped.startswith("</"):
flush()
continue
if ".md-button" in stripped:
act()
continue
if stripped.startswith("#"):
flush()
if run:
page.list_runs.append(run)
run = 0
kind = "heading"
current = [stripped.lstrip("#").strip()]
flush()
kind = "paragraph"
continue
if stripped.startswith("|"):
if kind != "table":
flush()
kind = "table"
if not re.fullmatch(r"\|[\s:|-]+\|?", stripped):
current.append(stripped.replace("|", " "))
continue
if kind == "table":
flush()
kind = "paragraph"
item = re.match(r"(?:[-*+]|\d+\.)\s+(.*)", stripped)
definition = re.match(r":\s+(.*)", stripped)
if item:
flush()
kind = "item"
run += 1
current = [item.group(1)]
continue
if definition:
flush()
if page.blocks and page.blocks[-1].kind == "paragraph":
page.blocks[-1].kind = "term"
kind = "definition"
current = [definition.group(1)]
continue
if kind == "item" and indent == 0:
flush()
kind = "paragraph"
if kind == "paragraph" and run:
page.list_runs.append(run)
run = 0
if not current and kind in ("item", "definition") and indent == 0:
kind = "paragraph"
current.append(stripped)
flush()
if run:
page.list_runs.append(run)
return page
def banned(block: Block, level: str) -> list[str]:
"""Filler, and jargon met without its meaning beside it."""
found = []
text = re.sub(r"`[^`]*`", "", block.text)
if block.kind == "heading" and text.rstrip().endswith("."):
return [] # a heading that quotes an installer message
if block.kind == "term":
return []
for sentence in split_sentences(text) or [text]:
lower = sentence.lower()
for phrase in FILLER:
if re.search(rf"(?<![\w-]){re.escape(phrase)}(?![\w-])", lower):
found.append(phrase)
terms = dict(JARGON)
if level == "start":
terms.update(START_ONLY)
for term, gloss in terms.items():
if not re.search(rf"(?<![\w-]){re.escape(term)}(?![\w-])", lower):
continue
if gloss and re.search(gloss, lower):
continue
if re.search(GENERIC_GLOSS.format(w=re.escape(term)), lower):
continue
if term in block.defines:
continue
found.append(term)
return found
def measure(page: Page) -> dict:
lengths = [count_words(s) for s in page.sentences()]
return {
"words": page.words(),
"before_action": page.before_action,
"max_block": max((count_words(b.text) for b in page.prose()), default=0),
"median_sentence": statistics.median(lengths) if lengths else 0,
"max_sentence": max(lengths, default=0),
}
def held_pages(docs: Path) -> list[tuple[str, str, int]]:
"""(relative path, level, ceiling) for every page the budgets hold."""
pages = [(name, "start", ceiling) for name, ceiling in START_PAGES.items()]
pages += [
(str(path.relative_to(docs)), "setup", SETUP_CEILING)
for path in sorted((docs / "platforms").glob("*.md"))
if path.name != "index.md"
and "\n## Where the files go" in path.read_text(encoding="utf-8")
]
pages.append(("platforms/index.md", "setup", SETUP_INDEX_CEILING))
return [(name, level, ceiling) for name, level, ceiling in pages if (docs / name).is_file()]
def _docs() -> Path:
if not (DOCS / "index.md").is_file():
if os.environ.get("CI"):
raise AssertionError("CI generates docs/ before the suite")
raise unittest.SkipTest("docs/ not generated (python scripts/generate_site.py)")
return DOCS
def _load(name: str) -> Page:
return parse((_docs() / name).read_text(encoding="utf-8"))
class CopyBudgets(unittest.TestCase):
def test_sentences_stay_short(self):
for name, level, _ceiling in held_pages(_docs()):
longest, median, _block, _before = BUDGETS[level]
page = _load(name)
with self.subTest(page=name):
long = [s for s in page.sentences() if count_words(s) > longest]
self.assertEqual(long, [], f"sentences over {longest} words")
self.assertLessEqual(measure(page)["median_sentence"], median)
def test_blocks_stay_short(self):
for name, level, _ceiling in held_pages(_docs()):
limit = BUDGETS[level][2]
page = _load(name)
with self.subTest(page=name):
long = [b.text for b in page.prose() if count_words(b.text) > limit]
self.assertEqual(long, [], f"blocks over {limit} words")
def test_the_first_action_comes_early(self):
for name, level, _ceiling in held_pages(_docs()):
limit = HOME_BEFORE_ACTION if name == "index.md" else BUDGETS[level][3]
page = _load(name)
if page.before_action is None:
continue # a page of definitions has nothing to run
with self.subTest(page=name):
self.assertLessEqual(page.before_action, limit)
def test_pages_stay_under_their_ceiling(self):
for name, _level, ceiling in held_pages(_docs()):
with self.subTest(page=name):
self.assertLessEqual(_load(name).words(), ceiling)
def test_lists_of_choices_stay_short(self):
for name, level, _ceiling in held_pages(_docs()):
with self.subTest(page=name):
self.assertLessEqual(max(_load(name).list_runs, default=0), LIST_ITEMS[level])
def test_headings_stay_short(self):
for name, _level, _ceiling in held_pages(_docs()):
page = _load(name)
long = [
b.text for b in page.blocks
if b.kind == "heading" and not b.text.endswith(".")
and count_words(re.sub(r"`[^`]*`", "", b.text)) > HEADING_WORDS
]
with self.subTest(page=name):
self.assertEqual(long, [])
class PlainWords(unittest.TestCase):
def test_no_filler_and_no_unexplained_jargon(self):
offenders = []
for name, level, _ceiling in held_pages(_docs()):
for block in _load(name).blocks:
if block.kind == "table":
continue
words = banned(block, level)
if words:
offenders.append(f"{name}: {sorted(set(words))} in: {block.text[:90]}")
self.assertEqual(offenders, [], "\n".join(offenders[:40]))
class TheMeasureItself(unittest.TestCase):
"""The parser decides every number above, so it is held on a sample."""
SAMPLE = (
"---\nlevel: start\n---\n\n# Title\n\nOne two three. Four five.\n\n"
"- Item **one**.\n- Item [two](x.md).\n\n```sh\nnot counted at all\n```\n\n"
"??? note \"Hidden\"\n\n Collapsed words here.\n\n"
"=== \"A\"\n\n First tab.\n\n=== \"B\"\n\n Second tab words.\n\n"
"| A | B |\n|---|---|\n| cell | cell |\n\n## Reference: deep\n\nNot read.\n"
)
def test_blocks_and_counts(self):
page = parse(self.SAMPLE)
kinds = [b.kind for b in page.blocks]
self.assertEqual(kinds, ["heading", "paragraph", "item", "item", "paragraph",
"paragraph", "paragraph", "table"])
self.assertEqual([b.hidden for b in page.blocks[5:7]], [False, True])
self.assertEqual(page.words(), 1 + 5 + 2 + 2 + 3 + 2)
self.assertEqual(page.before_action, 5 + 2 + 2)
self.assertEqual(page.list_runs, [2])
self.assertTrue(page.blocks[4].collapsed)
self.assertEqual(page.sentences()[:2], ["One two three.", "Four five."])
def test_a_gloss_allows_a_term(self):
self.assertEqual(banned(Block("paragraph", "Install Termux, a terminal app."), "start"), [])
self.assertEqual(banned(Block("paragraph", "Open Termux."), "start"), ["termux"])
self.assertEqual(banned(Block("paragraph", "Run `sudo` first."), "start"), [])
self.assertEqual(banned(Block("paragraph", "It is very easy."), "start"), ["easy", "very"])
self.assertEqual(banned(Block("paragraph", "The RetroArch pack."), "setup"), [])
def report(docs: Path) -> str:
rows = ["| Page | Words | Before action | Max block | Median / max sentence |",
"|---|---:|---:|---:|---|"]
for name, _level, _ceiling in held_pages(docs):
m = measure(parse((docs / name).read_text(encoding="utf-8")))
before = "none" if m["before_action"] is None else m["before_action"]
rows.append(f"| {name} | {m['words']} | {before} | {m['max_block']} | "
f"{m['median_sentence']:g} / {m['max_sentence']} |")
return "\n".join(rows)
if __name__ == "__main__":
if len(sys.argv) == 3 and sys.argv[1] == "--report":
print(report(Path(sys.argv[2])))
else:
unittest.main()
Reference in new issue View Git Blame Copy Permalink