chore: merge site and docs fixes, fifth pass

tests/test_audit_regressions.py

@@ -7574,5 +7574,70 @@ class TupleAnnotationsMatchTheirReturns(unittest.TestCase):
         self.assertEqual(mismatched, [])
 
 
+class PrboomPlusCitesFullPaths(unittest.TestCase):
+    """prboom-plus cited gl_hires.c, e6y.c, d_client.c and g_game.c bare:
+    none resolves against the repository, so the page showed them as plain
+    code with no link, and "behind GL_DOOM and PRBOOM_DEBUG" was read as a
+    seventeenth citation. Each name has one path in the tree at the pin."""
+
+    def test_every_cited_file_has_its_directory(self):
+        import yaml
+
+        from scripts.sourceref import source_ref_values, split_source_ref
+
+        profile = yaml.safe_load((ROOT / "emulators" / "prboom-plus.yml").read_text(encoding="utf-8"))
+        refs: list[str] = []
+
+        def walk(node):
+            if isinstance(node, dict):
+                for key, value in node.items():
+                    if key == "source_ref":
+                        refs.extend(r for _, r in source_ref_values(value))
+                    else:
+                        walk(value)
+            elif isinstance(node, list):
+                for item in node:
+                    walk(item)
+
+        walk(profile)
+        parts = [p for ref in refs for p in split_source_ref(ref)]
+        self.assertGreater(len(parts), 40)
+        for part in parts:
+            with self.subTest(path=part.path):
+                self.assertIsNotNone(part.start)
+                self.assertTrue(part.path.startswith("prboom2/"), part.path)
+
+
+class AutoFetchIssueBodyReadsAsWritten(unittest.TestCase):
+    """The --create-issues body was a template: Title Case headings, "The
+    following...", an exclamation mark, numbered orders to the reader, a
+    spaced dash, and hashes cut to 12 characters a contributor cannot
+    match a file against."""
+
+    ENTRY = {"name": "scph5501.bin", "system": "sony-playstation",
+             "sha1": "b056ee5a4d65937e1a3a17e1e78f3258ea49c38e",
+             "md5": "490f666e1afb15b7362b406ed1cea246"}
+
+    def test_full_hashes_and_no_template_phrasing(self):
+        from scripts import auto_fetch
+
+        body = auto_fetch.generate_issue_body([self.ENTRY], "RetroArch")
+        self.assertIn(f"`{self.ENTRY['sha1']}`", body)
+        self.assertIn(f"`{self.ENTRY['md5']}`", body)
+        self.assertNotIn("...", body)
+        self.assertNotIn("!", body)
+        self.assertNotRegex(body, r"(?i)the following|please| - |\bfork\b")
+        self.assertNotRegex(body, r"(?m)^\d+\. ")
+        for heading in re.findall(r"(?m)^#+ (.*)$", body):
+            later = [w for w in heading.split()[1:] if w != "RetroArch"]
+            self.assertEqual(later, [w.lower() for w in later], heading)
+
+    def test_the_docstring_says_nothing_is_posted(self):
+        from scripts import auto_fetch
+
+        self.assertNotIn("Create GitHub Issue", auto_fetch.__doc__)
+        self.assertIn("posts nothing", auto_fetch.__doc__)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_batocera_md5_list.py

@@ -345,7 +345,7 @@ class SeveralAcceptedMd5BecomeAList(unittest.TestCase):
         self.assertEqual(
             produced, LISTING.replace(f'"md5": "{C}"', f'"md5": [ "{C}", "{D}" ]')
         )
-        record = next(r for r in changes.records if r["kind"] == "hash_corrected")
+        record = next(r for r in changes.records if r["kind"] == "hash_updated")
         self.assertEqual((record["old"], record["new"]), (C, [C, D]))
         self.assertEqual(changes.lists, 1)
 
@@ -433,7 +433,7 @@ class SeveralAcceptedMd5BecomeAList(unittest.TestCase):
 class SitePageShowsAList(unittest.TestCase):
     def test_a_list_reads_as_any_of_its_values(self):
         line = describe_change({
-            "kind": "hash_corrected", "system": "psx", "name": "scph5500.bin",
+            "kind": "hash_updated", "system": "psx", "name": "scph5500.bin",
             "field": "md5", "old": C, "new": [C, D],
         })
         self.assertEqual(line, f"`psx/scph5500.bin` md5 `{C}` becomes `{C}` or `{D}`")

tests/test_contributing_text.py

@@ -37,7 +37,6 @@ class OneContributingText(unittest.TestCase):
         )
 
 
-
 class CoverageReadsTheGivenProfiles(unittest.TestCase):
     """generate_site --emulators-dir changed the emulator pages but not the
     platform coverage, which read ./emulators whatever was given."""
@@ -67,7 +66,7 @@ class BeforeOpeningAPullRequest(unittest.TestCase):
 
         text = (REPO_ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8")
         section = text.split("## Before opening a PR", 1)[1].split("\n## ", 1)[0]
-        block = _re.search(r"```bash\n(.*?)```", section, _re.S).group(1)
+        block = "".join(_re.findall(r"```bash\n(.*?)```", section, _re.S))
         for command in ("validate_pr.py --changed", "validate_schemas.py --source-only",
                         "unittest discover"):
             self.assertIn(command, block)
@@ -116,11 +115,41 @@ class BeforeOpeningAPullRequest(unittest.TestCase):
         line naming the old theme installed packages nothing reads."""
         text = generate_readme.generate_contributing()
         section = text.split("## Before opening a PR", 1)[1].split("\n## ", 1)[0]
-        self.assertIn('pip install pyyaml "jsonschema[format-nongpl]" markdown', section)
+        self.assertIn('pip install pyyaml "jsonschema[format-nongpl]==', section)
         self.assertNotIn("mkdocs", section)
         self.assertTrue((REPO_ROOT / "website" / "package-lock.json").is_file())
 
 
+class TheChecklistInstallsWhatCiInstalls(unittest.TestCase):
+    """The pip line named the packages without versions, and nothing said
+    that shellcheck, bubblewrap and a Python 3.8 with pyte come from the
+    system: their tests skipped locally and failed first in CI."""
+
+    def test_the_pip_lines_are_the_workflow_lines(self):
+        workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8")
+        text = generate_readme.generate_contributing()
+        section = text.split("## Before opening a PR", 1)[1].split("\n## ", 1)[0]
+        pinned = [line.split("run: ", 1)[1] for line in workflow.splitlines()
+                  if re.match(r"\s*run: (python3\.8 -m )?pip install .*==", line)
+                  and "pyte" in line]
+        self.assertEqual(len(pinned), 2)
+        for line in pinned:
+            with self.subTest(line=line):
+                self.assertIn(line, section)
+
+    def test_it_names_the_system_tools_ci_installs(self):
+        workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8")
+        text = generate_readme.generate_contributing()
+        section = text.split("## Before opening a PR", 1)[1].split("\n## ", 1)[0]
+        for tool, step in (("shellcheck", "shellcheck --version"),
+                           ("bwrap", "bwrap --ro-bind"),
+                           ("python3.8 -m unittest tests.test_install_tui",
+                            "python3.8 -m unittest tests.test_install_tui")):
+            with self.subTest(tool=tool):
+                self.assertIn(step, workflow)
+                self.assertIn(tool, section)
+
+
 class OneFileThroughTheBrowser(unittest.TestCase):
     """Adding one BIOS asked for a fork, a 5.5 GB clone, five pip packages,
     the data directories and the whole suite, and promised a PR comment that
@@ -174,5 +203,83 @@ class OneFileThroughTheBrowser(unittest.TestCase):
         ))
 
 
+class TheGuideRebuildsWhenItsPinsMove(unittest.TestCase):
+    """The checklist quotes validate.yml, so a pin bump there must redeploy
+    the guide and meet the README/CONTRIBUTING freshness check."""
+
+    def test_deploy_site_watches_the_validation_workflow(self):
+        import yaml
+
+        deploy = yaml.safe_load(
+            (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8"))
+        paths = deploy[True]["push"]["paths"]
+        self.assertIn(".github/workflows/validate.yml", paths)
+
+
+class OneCommandRunsTheChecks(unittest.TestCase):
+    """A contributor copied ten commands by hand before a pull request, and
+    skipping one (refresh_data_dirs) failed the pack tests with no hint."""
+
+    CHECK = re.compile(r"(?:xargs -0 |run: |^\s*)((?:python3?\.?8? -m unittest|python scripts/\w+\.py"
+                       r"|vermin|npm (?:ci|run build)))")
+
+    def test_every_check_ci_runs_is_a_step(self):
+        import pr_checks
+
+        workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8")
+        ran = {m.group(1) for line in workflow.splitlines()
+               if not line.strip().startswith("#") for m in [self.CHECK.search(line)] if m}
+        self.assertGreaterEqual(len(ran), 10)
+        steps = [step.command for step in pr_checks.STEPS]
+        for check in sorted(ran):
+            with self.subTest(check=check):
+                self.assertTrue(any(command.startswith(check) for command in steps))
+
+    def test_steps_keep_ci_order(self):
+        import pr_checks
+
+        workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8")
+        order = [workflow.index(key) for key in (
+            "refresh_data_dirs.py", "run: python scripts/generate_site.py",
+            "npm run build --prefix website", "python -m unittest discover",
+            "validate_site.py")]
+        self.assertEqual(order, sorted(order))
+        commands = [step.command for step in pr_checks.STEPS]
+        positions = [next(i for i, c in enumerate(commands) if key in c) for key in (
+            "refresh_data_dirs.py", "generate_site.py", "npm run build",
+            "unittest discover", "validate_site.py")]
+        self.assertEqual(positions, sorted(positions))
+
+    def test_the_guide_gives_the_runner_and_what_it_runs(self):
+        import pr_checks
+
+        text = generate_readme.generate_contributing()
+        section = text.split("## Before opening a PR", 1)[1].split("\n## ", 1)[0]
+        self.assertIn("python scripts/pr_checks.py\n", section)
+        for step in pr_checks.STEPS:
+            with self.subTest(step=step.command):
+                self.assertIn(step.command, section)
+
+    def test_a_missing_tool_is_reported_and_a_failure_does_not_stop_the_run(self):
+        import contextlib
+        import io
+
+        import pr_checks
+
+        steps = (
+            pr_checks.Step("python -c raise(SystemExit(3))", "fails"),
+            pr_checks.Step("no-such-tool-here --x", "absent", needs="no-such-tool-here",
+                           provider="nothing"),
+            pr_checks.Step("python -c pass", "passes"),
+        )
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            code = pr_checks.run(steps)
+        self.assertEqual(code, 1)
+        self.assertIn("FAILED   python -c raise(SystemExit(3)) (exit 3)", out.getvalue())
+        self.assertIn("NOT RUN  no-such-tool-here --x", out.getvalue())
+        self.assertIn("1/3 checks passed", out.getvalue())
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_doc_pages.py

@@ -1985,7 +1985,6 @@ class NoTextClaimsAScheduledScrape(unittest.TestCase):
             self.assertNotIn("list_platforms", workflow.read_text(encoding="utf-8"))
 
 
-
 class TheCatalogRowsStateFacts(unittest.TestCase):
     """tools.md said "redump.org serves stale DATs" and "Dat-o-Matic blocks
     automation", verdicts on two preservation projects, and gave Redump's
@@ -2003,6 +2002,100 @@ class TheCatalogRowsStateFacts(unittest.TestCase):
         self.assertIn("`/bios/<CODE>`", rows[0])
 
 
+class RegionFreeRuleNamesWhereItDoesNotHold(unittest.TestCase):
+    """advanced-usage.md said psxonpsp660.bin is declared [world] and always
+    survives --region. PCSX ReARMed declares it [japan], and on the platforms
+    where it is the only PlayStation core the filter drops it."""
+
+    def test_every_platform_that_drops_a_named_file_is_named(self):
+        from common import (list_registered_platforms, load_emulator_profiles,
+                            load_platform_config)
+        from region import WORLD, platform_region_index
+
+        section = _section(_page("advanced-usage.md"), "Region filtering")
+        bullet = next(p for p in section.split("\n- ") if "Region-free" in p)
+        files = re.findall(r"`([^`]+\.bin)`", bullet)
+        self.assertTrue(files)
+        profiles = load_emulator_profiles(str(ROOT / "emulators"))
+        for platform in list_registered_platforms(str(ROOT / "platforms")):
+            config = load_platform_config(platform, str(ROOT / "platforms"))
+            index = platform_region_index(config, profiles)
+            for name in files:
+                entry = index.get(name)
+                drops = bool(entry and entry["regions"] and WORLD not in entry["regions"]
+                             and not entry["has_untagged"])
+                if drops:
+                    with self.subTest(platform=platform, file=name):
+                        self.assertRegex(bullet, rf"(?i)\b{re.escape(platform)}\b")
+
+
+class BuildYourOwnPackSaysWhatTheCloneCosts(unittest.TestCase):
+    """The README sent a user after a minimal pack to "Clone the repo", with no
+    command and no size, and the clone runs to several gigabytes."""
+
+    def test_readme_gives_the_clone_command_and_its_size(self):
+        import generate_readme
+
+        db = json.loads((ROOT / "database.json").read_text(encoding="utf-8"))
+        line = generate_readme.clone_line(db)
+        self.assertIn("git clone --depth 1", line)
+        self.assertRegex(line, r"over \d+(\.\d)? GB")
+        section = _section((ROOT / "README.md").read_text(encoding="utf-8"),
+                           "Build your own pack")
+        self.assertIn("git clone --depth 1", section)
+
+    def test_advanced_usage_names_the_shallow_clone(self):
+        self.assertIn("--depth 1", _page("advanced-usage.md").split("## Custom packs")[0])
+
+
+class LandingAChangeNamesTheCommitRules(unittest.TestCase):
+    """The page left out the review-tag rule scripts/commit_rules.py
+    enforces, and never said where the rules live or how a clone runs them:
+    the hook is not versioned."""
+
+    def _landing(self) -> str:
+        return _section(_page("testing-guide.md"), "Landing a change")
+
+    def test_every_refusal_is_on_the_page(self):
+        import commit_rules
+
+        section = self._landing()
+        words = re.search(r"\\b\((.*?)\)", commit_rules.REFUSED_WORDS.pattern).group(1)
+        from tests.test_prose_style import NoMarketingVocabulary
+
+        for word in words.replace("|", " ").split():
+            if NoMarketingVocabulary.WORDS.search(word):
+                # Named by "the marketing vocabulary above": spelled out,
+                # the page would fail that test.
+                continue
+            with self.subTest(word=word):
+                self.assertIn(word, section.lower())
+        self.assertIn("which review asked for it", section)
+        self.assertIn("scripts/commit_rules.py", section)
+
+    def test_the_hook_command_on_the_page_refuses_a_review_tag(self):
+        import subprocess
+        import tempfile
+
+        block = re.search(r"```bash\n(.*?)```", self._landing(), re.S).group(1)
+        with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch:
+            clone = Path(scratch)
+            (clone / ".git" / "hooks").mkdir(parents=True)
+            (clone / "scripts").mkdir()
+            (clone / "scripts" / "commit_rules.py").write_bytes(
+                (ROOT / "scripts" / "commit_rules.py").read_bytes())
+            subprocess.run(["sh", "-c", block], cwd=clone, check=True)
+            message = clone / "MSG"
+            message.write_text("fix: address review 3 notes\n", encoding="utf-8")
+            refused = subprocess.run([str(clone / ".git" / "hooks" / "commit-msg"), "MSG"],
+                                     cwd=clone, capture_output=True, text=True)
+            message.write_text("fix: say what the change does\n", encoding="utf-8")
+            accepted = subprocess.run([str(clone / ".git" / "hooks" / "commit-msg"), "MSG"],
+                                      cwd=clone, capture_output=True, text=True)
+        self.assertEqual(refused.returncode, 1, refused.stderr)
+        self.assertIn("which review asked for it", refused.stderr)
+        self.assertEqual(accepted.returncode, 0, accepted.stderr)
+
+
 if __name__ == "__main__":
     unittest.main()
-

tests/test_doc_surfaces.py

@@ -436,8 +436,6 @@ class PagesOpenOnTheirContent(unittest.TestCase):
         self.assertEqual(announcing, [])
 
 
-
-
 class CoreCountsReadFromTheData(unittest.TestCase):
     """The cores page said "623 cores across 18 platforms": that was every
     profile, and 91 of them no platform runs."""
@@ -611,5 +609,31 @@ class EveryOptionExists(unittest.TestCase):
                  if (bad := unknown_options(text))}
         self.assertEqual(wrong, {})
 
+
+class ContributorChecksFetchThePlatformsFiles(unittest.TestCase):
+    """validate.yml fetches each platform's own file before the suite. The
+    checklist left it out, so the exporter fidelity cases skipped on a
+    contributor's clone and first ran, red, in CI."""
+
+    def test_the_pr_checklist_fetches_them_before_the_tests(self):
+        workflow = _read(".github/workflows/validate.yml")
+        self.assertLess(
+            workflow.index("python scripts/export_native.py --all --fetch"),
+            workflow.index("python -m unittest discover"),
+        )
+        for text in (
+            contributing_text(lambda page: page, "# C", "here"),
+            _read("CONTRIBUTING.md"),
+        ):
+            block = _section(text, "## Before opening a PR")
+            self.assertIn("python scripts/export_native.py --all --fetch", block)
+            self.assertLess(
+                block.index("export_native.py"), block.index("unittest discover")
+            )
+
+    def test_the_testing_guide_names_the_command(self):
+        self.assertIn("export_native.py --all --fetch", _read("wiki/testing-guide.md"))
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_export_counts.py

@@ -1083,7 +1083,7 @@ class AnExtendedListIsReported(unittest.TestCase):
                 self.assertEqual(fe.hashes("md5"), [A, B])
                 self.assertEqual(
                     [(r["kind"], r["field"], r["old"], r["new"]) for r in records],
-                    [("hash_corrected", "md5", A, [A, B])],
+                    [("hash_updated", "md5", A, [A, B])],
                 )
 
     def test_an_unchanged_value_reports_nothing(self):

tests/test_export_report.py

@@ -29,8 +29,8 @@ def _report(platform: str, files: list[str], **changes: object) -> dict:
         "originals": {f: f"https://example.org/{platform}/{f}" for f in files},
         "files": files,
         "summary": "3 kept, 1 file added, 7 hashes updated to the emulator source",
-        "hashes_corrected": [], "hashes_filled": [], "sizes_corrected": [],
-        "sizes_removed": [], "hashes_cleared": [], "requirements_corrected": 0,
+        "hashes_updated": [], "hashes_filled": [], "sizes_updated": [],
+        "sizes_removed": [], "hashes_cleared": [], "required_aligned": 0,
         "withheld": [], "invalid": [],
     }
     report.update(changes)
@@ -58,8 +58,8 @@ class ExportReportIsWrittenAndRead(unittest.TestCase):
         body = source[source.index("def export_platform("):source.index("def write_report(")]
         call = body[body.index("write_report("):]
         for key, value in (
-            ("hashes_corrected", "applied"), ("hashes_filled", "filled"),
-            ("sizes_corrected", "resized"), ("sizes_removed", "unsized"),
+            ("hashes_updated", "applied"), ("hashes_filled", "filled"),
+            ("sizes_updated", "resized"), ("sizes_removed", "unsized"),
             ("hashes_cleared", "cleared"), ("invalid", "issues"),
         ):
             self.assertIn(f'"{key}": {value},', call)
@@ -73,7 +73,7 @@ class CorrectedFilesPage(unittest.TestCase):
     def setUp(self):
         corrected = [f"sys/file{i}.bin md5" for i in range(12)]
         self.reports = [
-            _report("recalbox", ["es_bios.xml"], hashes_corrected=corrected),
+            _report("recalbox", ["es_bios.xml"], hashes_updated=corrected),
             _report("retropie", ["scriptmodules/a.sh", "scriptmodules/b.sh"],
                     invalid=["bash -n failed on a.sh"],
                     withheld=["a size the format has no field for"]),
@@ -154,7 +154,7 @@ class ThePageAddressesMaintainersAsPeers(unittest.TestCase):
         from siteexports import change_text
 
         report = _report("batocera", ["batocera-systems"],
-                         hashes_corrected=["a md5", "b md5"], requirements_corrected=3)
+                         hashes_updated=["a md5", "b md5"], required_aligned=3)
         report["counts"] = {"kept": 692, "added": 253, "not_added": 10, "not_kept": 0}
         self.assertEqual(
             change_text(report),
@@ -163,7 +163,7 @@ class ThePageAddressesMaintainersAsPeers(unittest.TestCase):
         )
 
     def test_no_wording_that_judges_the_original(self):
-        report = _report("batocera", ["batocera-systems"], hashes_corrected=["a md5"])
+        report = _report("batocera", ["batocera-systems"], hashes_updated=["a md5"])
         report["counts"] = {"kept": 1, "added": 0, "not_added": 4, "not_kept": 1}
         page = generate_exports_page([report], {}).lower()
         for word in ("their file", "theirs", "corrected", "wrong", "drift", "left out"):
@@ -243,13 +243,29 @@ class TheRunReportStatesWhatTheFileCarries(unittest.TestCase):
         for text in [report["summary"], *messages]:
             for word in self.JUDGING:
                 self.assertNotIn(word, text.lower())
+        # The report is served as api/v1/exports/<platform>.json, keys and
+        # change kinds included: "hash_corrected" sat on 130 Recalbox lines.
+        names = [*report, *(c["kind"] for c in report["changes"])]
+        self.assertIn("hash_updated", names)
+        for name in names:
+            for word in self.JUDGING:
+                self.assertNotIn(word, name)
+
+    def test_the_committed_reports_use_the_same_names(self):
+        import json  # noqa: PLC0415
+
+        for path in sorted((ROOT / "exports").glob("*.json")):
+            report = json.loads(path.read_text(encoding="utf-8"))
+            names = [*report, *(c["kind"] for c in report.get("changes") or [])]
+            with self.subTest(report=path.name):
+                self.assertFalse([n for n in names if "corrected" in n])
 
 
 UPSTREAM_CACHE = ROOT / ".cache" / "upstream-native"
 _DAT_ROM = re.compile(r"^\trom \( name (\"[^\"]+\"|\S+) (.*) \)$")
 _DAT_FIELDS = {"size": "size", "crc": "crc32", "md5": "md5", "sha1": "sha1"}
 _HASH_KINDS = {
-    "hash_corrected", "hash_filled", "hash_cleared", "size_corrected", "size_removed",
+    "hash_updated", "hash_filled", "hash_cleared", "size_updated", "size_removed",
 }
 
 
@@ -366,7 +382,7 @@ class ReportedChangesAreTheDiff(unittest.TestCase):
             ((c["system"], path_of[(c["system"], c["name"], c["destination"])]),
              c["field"], _norm(c["old"]), _norm(c["new"]))
             for c in changes
-            if c["kind"] in _HASH_KINDS or c["kind"] == "requirement_corrected"
+            if c["kind"] in _HASH_KINDS or c["kind"] == "required_aligned"
         }
         self.assertTrue(changed, "nothing changed, the comparison proves nothing")
         self.assertEqual(reported, changed)
@@ -420,10 +436,10 @@ class ThePageShowsEachValueBeforeAndAfter(unittest.TestCase):
 
         self.report = _report(
             "recalbox", ["es_bios.xml"],
-            hashes_corrected=["psx/scph5501.bin md5"], requirements_corrected=1,
+            hashes_updated=["psx/scph5501.bin md5"], required_aligned=1,
             changes=[
-                change("hash_corrected", "md5", "5626" + "0" * 28, "c523" + "1" * 28),
-                change("requirement_corrected", "required", True, False),
+                change("hash_updated", "md5", "5626" + "0" * 28, "c523" + "1" * 28),
+                change("required_aligned", "required", True, False),
                 change("hash_filled", "sha1", None, "f" * 40),
             ],
         )

tests/test_exporters.py

@@ -1451,7 +1451,7 @@ class RecalboxExport(unittest.TestCase):
         self.assertNotEqual(element.get("mandatory"), "false")
         self.assertEqual(
             [c for c in landed_changes(exporter, systems).records
-             if c["kind"] == "requirement_corrected"],
+             if c["kind"] == "required_aligned"],
             [],
         )
 

tests/test_measurements.py

@@ -470,6 +470,22 @@ class SitePages(unittest.TestCase):
     def item(self, **values: object) -> dict:
         return {**self.ITEM, **values}
 
+    def test_the_data_page_claims_no_link_an_emulator_page_lacks(self) -> None:
+        """The Data page said each per-emulator database is "linked from its
+        page". An emulator page shows the proof lines, which link the record
+        lines and never the database."""
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as scratch:
+            path = Path(scratch) / "m.sqlite"
+            connection = sqlite3.connect(path)
+            connection.execute("CREATE TABLE meta (key TEXT)")
+            connection.commit()
+            connection.close()
+            text = "\n".join(sm.data_page_section(path))
+        proofs = "\n".join(sm.proof_section([self.item()]))
+        self.assertNotIn("downloads/measurements/", proofs)
+        self.assertNotIn("linked from its page", text)
+        self.assertIn("downloads/measurements/<emulator>.sqlite", text)
+
     URL = f"{REPO_URL}/blob/{'e' * 40}/measurements/beetle_psx.jsonl"
 
     def record(self, line: int, proves=(), shown=None, observed=None) -> dict:

tests/test_prose_style.py

@@ -157,6 +157,17 @@ class NoGeneratedTypography(unittest.TestCase):
                     offenders.append(f"{path.relative_to(ROOT)}:{number}: {line.strip()[:100]}")
         self.assertEqual(offenders, [], "\n".join(offenders[:40]))
 
+    def test_no_double_hyphen_dash_in_profiles(self):
+        """GLYPHS sees U+2014, not its ASCII spelling: "No separate upstream --
+        written directly for the libretro API" reached the cruzes page as is."""
+        dash = re.compile(r"[A-Za-z0-9)] -- [A-Za-z0-9(]")
+        offenders = []
+        for path in sorted((ROOT / "emulators").glob("*.yml")):
+            for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+                if dash.search(BACKTICKS.sub("", line)):
+                    offenders.append(f"{path.relative_to(ROOT)}:{number}: {line.strip()[:100]}")
+        self.assertEqual(offenders, [], "\n".join(offenders))
+
 
 class NoMarketingVocabulary(unittest.TestCase):
     """The project bans these words in published text, and nothing checked

tests/test_site_emulator_page.py

@@ -150,6 +150,15 @@ class WhatASearchSnippetSays(unittest.TestCase):
         self.assertTrue(plain.startswith(text[:-3]))
         self.assertIn(plain[len(text) - 3], " ,:")
 
+    def test_a_link_whose_text_holds_brackets_keeps_its_text_only(self):
+        """The Pioneer meta description published the whole link, anchor
+        included: the text `[BIOS] ...bin` stopped the pattern at its "]"."""
+        self.assertEqual(
+            gs._plain_markdown("**Start with** [`[BIOS] LaserActive PAC-N1 (Japan) (v1.02).bin`]"
+                               "(#sha1-6973e2593e66). ares reads it."),
+            "Start with [BIOS] LaserActive PAC-N1 (Japan) (v1.02).bin. ares reads it.")
+        self.assertEqual(gs._plain_markdown("[a [b] c](x) and [d](y)"), "a [b] c and d")
+
     def test_code_keeps_its_underscores(self):
         self.assertEqual(gs._plain_markdown("only `validate_schemas.py` and _this_"),
                          "only validate_schemas.py and this")

tests/test_site_inputs.py

@@ -87,11 +87,48 @@ class TheGivenProfilesAndPlatforms(_Inputs):
         self.assertEqual(sitediagrams.registry()["retroarch"].get("release_pack"), False)
 
 
+class TheCollectionDoesNotDependOnTheWorkingDirectory(unittest.TestCase):
+    """database.json names its bios folder "bios", and the collection model
+    read it from the working directory: started from tmp/, it found no copy
+    on disk and dropped five system sections and 2,500 collection paths."""
+
+    def test_same_sections_and_paths_from_another_directory(self) -> None:
+        import json
+
+        import sitecollection
+
+        db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+        here = sitecollection.collection_paths(db)
+        places = sitecollection._copy_places(db)
+        elsewhere = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None))
+        self.addCleanup(shutil.rmtree, elsewhere)
+        cwd = os.getcwd()
+        os.chdir(elsewhere)
+        self.addCleanup(os.chdir, cwd)
+        self.assertEqual(sitecollection.collection_paths(db), here)
+        self.assertEqual(sitecollection._copy_places(db), places)
+        self.assertGreater(sum(map(len, here.values())), len(here))
+
+    def test_the_bios_folder_sits_beside_the_given_database(self) -> None:
+        import siteinputs
+
+        siteinputs.use(siteinputs.SiteInputs(db=Path("/elsewhere/database.json")))
+        self.addCleanup(siteinputs.use, siteinputs.SiteInputs())
+        self.assertEqual(siteinputs.bios_dir({"bios_dir": "bios"}), Path("/elsewhere/bios"))
+        self.assertEqual(siteinputs.bios_dir({"bios_dir": "/abs/bios"}), Path("/abs/bios"))
+
+    def test_the_image_tools_do_not_load_the_page_generator(self) -> None:
+        for module in ("make_og_cards.py", "fetch_site_images.py", "siterender.py"):
+            with self.subTest(module=module):
+                source = (REPO_ROOT / "scripts" / module).read_text(encoding="utf-8")
+                self.assertNotIn("from generate_site import", source)
+
+
 class NoPageModuleReadsTheRepositoryInputs(unittest.TestCase):
     """A helper that opens the repository's database, profiles or platform
     files itself bypasses the flags. siteinputs.current() is the one source."""
 
-    MODULES = ("generate_site.py", "siterender.py", "sitediagrams.py")
+    MODULES = ("generate_site.py", "siterender.py", "sitediagrams.py", "sitecollection.py")
     INPUTS = {"database.json", "emulators", "platforms"}
 
     def test_no_literal_input_path(self) -> None:

tests/test_site_privacy.py

@@ -51,6 +51,25 @@ class TheCounterAsksNothingOfTheDevice(unittest.TestCase):
         self.assertIn("countEvent(`search-none/${searchWords(query)}`)", self.code)
         self.assertIn("message.startsWith('No results for')", self.code)
 
+    def _shape(self, name: str):
+        """A search-shape predicate of count.ts, its regexes run by Python
+        (the counter's patterns use no syntax the two read differently)."""
+        body = re.search(rf"const {name} = \(query: string\): boolean =>(.*?);\n", self.code, re.S).group(1)
+        patterns = [re.compile(p) for p in re.findall(r"/(\^.*?\$|\[[^/]*?\]|\\d)/\.test\(query\)", body)]
+        digits = "(query.match(/\\d/g) ?? []).length <= 4" in body
+        return lambda q: all(p.search(q) for p in patterns) and (
+            not digits or len(re.findall(r"\d", q)) <= 4)
+
+    def test_a_number_alone_is_never_sent(self) -> None:
+        """An 8-digit phone number is valid hex and went out as a hash."""
+        shapes = [self._shape(n) for n in ("isHash", "isFileName", "isFileStem")]
+        for query in ("55512345", "91234567", "0612345678", "+33612345678"):
+            with self.subTest(query=query):
+                self.assertFalse(any(shape(query) for shape in shapes))
+        for query in ("b4430a3b", "scph5501", "scph5501.bin", "a" * 40):
+            with self.subTest(query=query):
+                self.assertTrue(any(shape(query) for shape in shapes))
+
     def test_one_endpoint(self) -> None:
         self.assertEqual(set(re.findall(r"https://[\w.-]+", self.code)),
                          {"https://retrobios.goatcounter.com"})
@@ -72,6 +91,19 @@ class ThePageSaysWhatIsCounted(unittest.TestCase):
             with self.subTest(said=said):
                 self.assertIn(said, page)
 
+    def test_the_page_names_what_the_counter_cannot_tell_apart(self) -> None:
+        """The page left out outbound clicks, which count.ts sends, said the
+        referrer went by domain when the origin goes, and promised a search
+        word goes out only as a file name while a nickname with digits does."""
+        page = PRIVACY.read_text(encoding="utf-8")
+        code = COUNTER.read_text(encoding="utf-8")
+        self.assertIn("`out/${url.hostname}`", code)
+        self.assertIn("link to another site", page)
+        self.assertIn("from.origin", code)
+        self.assertIn("`https://www.google.com`", page)
+        self.assertIn("`dupont1985`", page)
+        self.assertIn("phone number, never is", page)
+
     def test_the_installer_has_no_counter(self) -> None:
         installer = (REPO_ROOT / "install.py").read_text(encoding="utf-8")
         self.assertNotIn("goatcounter", installer.lower())

tests/test_site_reference.py

@@ -1104,7 +1104,7 @@ class SourceLinksOnlyWhereTheFileIs(unittest.TestCase):
         profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40}
         text = sitesources._source_ref_markdown(
             profile, "Nesoid 2.5.0 libnes.so 0x455a4-0x45694 (setter), 0x8094-0x80ac")
-        self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so`, "
+        self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so` (setter), "
                                "offset `0x8094-0x80ac` in `Nesoid 2.5.0 libnes.so`")
 
     def test_only_a_recorded_path_links(self):

tests/test_site_render.py

@@ -257,6 +257,47 @@ class AnAliasTreeIsListedOnItsParentsPage(unittest.TestCase):
         self.assertEqual([t["ref"] for t in trees], ["freeintv-overlays"])
 
 
+class CitationsKeepTheirAnnotations(unittest.TestCase):
+    """split_source_ref drops annotations for profile_sync, and the cards
+    reused it: abuse's two lsf lines lost "(lsf default)" and "(lsf
+    override)", the only words saying which one wins."""
+
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)
+    def test_each_part_shows_what_the_profile_wrote_beside_it(self):
+        import sitesources
+
+        profile = {"source": "https://github.com/Xenoveritas/abuse",
+                   "source_commit": "a" * 40}
+        ref = ("src/loader2.cpp:284 (lsf default), src/netcfg.cpp:204 (lsf override), "
+               "libretro.cpp:1520-1521 candidates_a1200, 30-31 (see <system_dir>), src/x.c:1")
+        (label, items), = sitesources.source_ref_items(profile, ref)
+        texts = [text for text, _role in items]
+        self.assertTrue(texts[0].endswith(") (lsf default)"), texts[0])
+        self.assertTrue(texts[1].endswith(") (lsf override)"), texts[1])
+        self.assertTrue(texts[2].endswith(") candidates_a1200"), texts[2])
+        self.assertTrue(texts[3].endswith(") (see &lt;system_dir>)"), texts[3])
+        self.assertTrue(texts[4].endswith(")"), texts[4])
+
+    def test_the_annotation_is_what_follows_the_location(self):
+        from sourceref import ref_annotation
+
+        self.assertEqual(ref_annotation("munt ROMInfo.cpp:206-213"), "")
+        self.assertEqual(ref_annotation("source atari800/src/sysrom.c:91"), "")
+        self.assertEqual(ref_annotation("doc/notes.txt (read at boot)"), "(read at boot)")
+        self.assertEqual(ref_annotation("a.c (see b.c:12)"), "(see b.c:12)")
+        self.assertEqual(ref_annotation("0x53c338-0x53c36f (dreamBios row)"), "(dreamBios row)")
+
+    def test_a_binary_citation_keeps_its_annotation(self):
+        import sitesources
+
+        (_label, items), = sitesources.source_ref_items(
+            {}, "demul.exe 0x6b4d80-0x6b4ea3 (dc table), 0x53ca4d-0x53ca7e (load, fatal)")
+        self.assertEqual([text for text, _ in items], [
+            "offset `0x6b4d80-0x6b4ea3` in `demul.exe` (dc table)",
+            "offset `0x53ca4d-0x53ca7e` in `demul.exe` (load, fatal)",
+        ])
+
+
 class ForgesUpstreamDoesNotRead(unittest.TestCase):
     """Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
     Heritage showed no revision, and their citations were plain code."""