From 78e22851ddf8cbfba3f381ab4c6849f5d8c4270c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:09:47 +0100 Subject: [PATCH] chore: guard the site against personal pronouns --- tests/test_site_pronouns.py | 204 ++++++++++++++++++++++++++++++++++++ wiki/testing-guide.md | 8 +- 2 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 tests/test_site_pronouns.py diff --git a/tests/test_site_pronouns.py b/tests/test_site_pronouns.py new file mode 100644 index 000000000..44283b699 --- /dev/null +++ b/tests/test_site_pronouns.py @@ -0,0 +1,204 @@ +"""The site speaks without personal pronouns. + +Every page is written as an instruction or a plain statement: "Open the +Start menu.", "The installer finds the BIOS folder." No I, we, you or their +forms in the wiki sources, the pages the generators write, the README and +CONTRIBUTING, llms.txt or the agent skill. Code, URLs, markup and anchor ids +are not prose. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import html +import re +import subprocess +import sys +import unittest +from functools import cache +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DOCS = ROOT / "website" / "src" / "content" / "docs" + +PRONOUN = re.compile( + r"(?", re.S) +SCRIPT_STYLE = re.compile(r"<(script|style|pre|code)\b.*?", re.S | re.I) +TAG = re.compile(r"]*>") +ENTITY = re.compile(r"&#(?:\d+|x[0-9a-fA-F]+);") +URL = re.compile(r"(?:https?|ftp|smb)://\S+|\bwww\.\S+") +LINK_TARGET = re.compile(r"\]\([^)\s]*(?:\s+\"[^\"]*\")?\)|^\[[^\]]+\]:\s*\S+.*$", re.M) +ANCHOR = re.compile(r"\{[^{}\n]*#[^{}\n]*\}") +PATHLIKE = re.compile(r"\S*[/\\]\S*") +# File names, domains and dotted initials: baserom.us.z64, delete.me, A.I.R. +DOTTED = re.compile(r"[\w-]+(?:\.[\w-]+)+\.?") +# A list of single letters or digits (D, F, I, P), or of lowercase option +# values in parentheses (euro, us, asia), names keys, not people. +LETTERS = re.compile(r"(? str: + return re.sub(r"[^\n]", " ", match[0]) + + +def prose(text: str) -> str: + """The text of a page with everything that is not prose blanked.""" + for pattern in (HTML_COMMENT, SCRIPT_STYLE, TAG): + text = pattern.sub(_blank, text) + # _ and ' spell a file name or a name: read them as characters. + text = ENTITY.sub(lambda m: html.unescape(m[0]), text) + for pattern in ( + *_SPACED, FENCED, INLINE_CODE, LINK_TARGET, URL, ANCHOR, PATHLIKE, + TERMS, *_NAMES, DOTTED, LETTERS, VALUES, LETTER_HEADING, COUNTRY, + ): + text = pattern.sub(_blank, text) + return text + + +def pronouns(text: str) -> list[tuple[int, str]]: + """(line number, line) for each prose line holding a personal pronoun.""" + lines = text.splitlines() + return [ + (number, lines[number - 1].strip()) + for number, line in enumerate(prose(text).splitlines(), 1) + if PRONOUN.search(line) + ] + + +def _hits(paths: list[Path]) -> list[str]: + return [ + f"{path.relative_to(ROOT)}:{number}: {line}" + for path in paths + for number, line in pronouns(path.read_text(encoding="utf-8")) + ] + + +@cache +def _generated() -> bool: + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "generate_site.py")], + cwd=ROOT, capture_output=True, text=True, check=False, + ) + return result.returncode == 0 + + +class Scanner(unittest.TestCase): + def test_code_urls_and_anchors_are_not_prose(self): + text = ( + "Run `install.py --help` first.\n" + "```\nyou are here\n```\n" + "See https://example.org/your/page and [a link](our-page.md).\n" + "## Get the files { #get-your-files }\n" + 'Open the menu\n' + "A US BIOS, Quake II I mean.\n" + ) + self.assertEqual(pronouns(text), [(8, "A US BIOS, Quake II I mean.")]) + + def test_each_form_is_found_in_any_case(self): + for word in ("I", "me", "My", "mine", "we", "Us", "OUR", "you", "Yours", + "yourselves", "you're", "I'm"): + with self.subTest(word=word): + self.assertTrue(pronouns(f"Then {word} copy it.")) + + def test_a_word_holding_the_letters_passes(self): + self.assertEqual(pronouns("Usage of the menu, mine-craft, youth."), []) + + +class SitePronouns(unittest.TestCase): + def test_wiki_sources(self): + hits = _hits(sorted((ROOT / "wiki").glob("*.md"))) + self.assertEqual(hits, [], "\n".join(hits)) + + def test_readme_and_contributing(self): + hits = _hits([ROOT / "README.md", ROOT / "CONTRIBUTING.md"]) + self.assertEqual(hits, [], "\n".join(hits)) + + def test_generated_pages(self): + if not _generated(): + self.fail("scripts/generate_site.py failed") + pages = sorted(p for p in DOCS.rglob("*") if p.suffix in (".md", ".mdx")) + self.assertTrue(pages) + hits = _hits(pages) + self.assertEqual(hits, [], "\n".join(hits[:200])) + + def test_agent_files(self): + if not _generated(): + self.fail("scripts/generate_site.py failed") + public = ROOT / "website" / "public" + files = [public / "llms.txt", *sorted((public / "skills").rglob("*.md"))] + self.assertTrue(all(path.is_file() for path in files)) + hits = _hits(files) + self.assertEqual(hits, [], "\n".join(hits)) + + def test_every_allowed_quote_says_why(self): + for quoted, reason in {**NAMES, **INSTALLER, **SOURCE}.items(): + with self.subTest(quoted=quoted): + self.assertTrue(reason.strip()) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 187ad6499..23197370a 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -69,7 +69,7 @@ Every module in `tests/`, by the area it guards. | Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_agents`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_demo`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_inputs`, `test_site_insights`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_privacy`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_steps`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_agents`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_demo`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_inputs`, `test_site_insights`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_privacy`, `test_site_pronouns`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_steps`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | | Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` | ## Test architecture @@ -219,7 +219,7 @@ disambiguated by a hash suffix on purpose. ## How to add a test 1. **Pick the right category.** Find the number range that matches the - subsystem you are testing. If none fits, start a new range after the last + subsystem under test. If none fits, start a new range after the last existing one. 2. **Create synthetic fixtures.** Write the minimum YAML configs and fake @@ -235,7 +235,7 @@ disambiguated by a hash suffix on purpose. method, file counts, or pack contents. Avoid brittle assertions on log output or formatting. -5. **Run the full suite.** After adding your test, run `python -m unittest +5. **Run the full suite.** After adding the test, run `python -m unittest discover -s tests -t . -v` to verify nothing else broke. A new module starts with `import tests` right after @@ -497,7 +497,7 @@ chmod +x .git/hooks/commit-msg ``` Files are staged by name, never with `git add -A` or `git add .`, which sweep in -deletions and work that is not yours. Generated artifacts (`database.json`, +deletions and other people's work. Generated artifacts (`database.json`, `retired.json`, `derivable.json`, `install/*.json`, `README.md`, `CONTRIBUTING.md`, `exports/`, the generated tables and run samples of `wiki/installer.md`) are committed with the data they derive from, never alone: a manifest committed before the