feat: add site insights report

Search Console queries by page, file names searched against the
collection, GoatCounter totals and GitHub traffic in one Markdown report.

scripts/site_insights.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""What readers look for and do not find, from totals only.
+
+Four sources, none of which names a person:
+
+- Google Search Console: the searches that showed the site, by page.
+- GoatCounter: page views, the domains readers came from, and the actions
+  website/src/scripts/count.ts counts (searches that found nothing,
+  "Did this page help?", downloads, broken links). wiki/privacy.md says
+  what is sent.
+- GitHub: repository traffic (14 days) and release download counts.
+- The collection: which searched file names it holds, and which no
+  emulator profile declares.
+
+The report is Markdown on stdout. Credentials stay outside the repository:
+
+    ~/.config/retrobios/gsc-service-account.json   Search Console, read only
+    ~/.config/retrobios/goatcounter-api-token.txt   GoatCounter, read only
+    GITHUB_TOKEN or gh auth token                   GitHub
+
+Usage:
+    python scripts/site_insights.py
+    python scripts/site_insights.py --days 90 --output report.md
+    python scripts/site_insights.py --skip gsc,github
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.parse
+import urllib.request
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(__file__))
+from common import github_token, load_database, load_emulator_profiles, run_cli
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+CONFIG = Path(os.environ.get("RETROBIOS_CONFIG", Path.home() / ".config" / "retrobios"))
+SITE_PROPERTY = "https://abdess.github.io/retrobios/"
+GOATCOUNTER = "https://retrobios.goatcounter.com/api/v0"
+REPOSITORY = "Abdess/retrobios"
+SOURCES = ("gsc", "goatcounter", "github")
+
+# A word of a search that names a file: "scph5501.bin", "prod.keys".
+FILE_TOKEN = re.compile(r"\b[\w.+-]+\.(?:bin|rom|zip|pup|keys|img|dat|bios|ic\d+|u\d+|sys|nvm|mec|"
+                        r"rom1|j64|pce|col|sms|gb|gbc|gba|nds|cfg|chd|iso|txt|fw|dll|xml|yaml)\b", re.I)
+
+
+@dataclass(frozen=True)
+class Query:
+    text: str
+    page: str
+    clicks: int
+    impressions: int
+    position: float
+
+    @property
+    def ctr(self) -> float:
+        return self.clicks / self.impressions if self.impressions else 0.0
+
+
+# Reading the sources
+
+
+def _get_json(url: str, headers: dict[str, str] | None = None, body: dict | None = None) -> dict | list:
+    data = json.dumps(body).encode() if body is not None else None
+    request = urllib.request.Request(url, data=data, headers={
+        "User-Agent": "retrobios-site-insights", **({"Content-Type": "application/json"} if data else {}),
+        **(headers or {})})
+    with urllib.request.urlopen(request, timeout=60) as response:
+        return json.load(response)
+
+
+def _google_token(key_file: Path) -> str:
+    """An access token for a service account, signed by openssl: the JWT
+    grant needs RS256 and the standard library has no RSA."""
+    key = json.loads(key_file.read_text(encoding="utf-8"))
+    now = int(time.time())
+
+    def b64(raw: bytes) -> str:
+        return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
+
+    claims = {"iss": key["client_email"], "scope": "https://www.googleapis.com/auth/webmasters.readonly",
+              "aud": key["token_uri"], "iat": now, "exp": now + 3600}
+    unsigned = f"{b64(json.dumps({'alg': 'RS256', 'typ': 'JWT'}).encode())}.{b64(json.dumps(claims).encode())}"
+    with tempfile.TemporaryDirectory() as scratch:
+        pem = Path(scratch) / "key.pem"
+        pem.touch(mode=0o600)
+        pem.write_text(key["private_key"], encoding="utf-8")
+        signature = subprocess.run(["openssl", "dgst", "-sha256", "-sign", str(pem)],
+                                   input=unsigned.encode(), capture_output=True, check=True).stdout
+    body = urllib.parse.urlencode({"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+                                   "assertion": f"{unsigned}.{b64(signature)}"}).encode()
+    request = urllib.request.Request(key["token_uri"], data=body)
+    with urllib.request.urlopen(request, timeout=60) as response:
+        return json.load(response)["access_token"]
+
+
+def fetch_queries(start: dt.date, end: dt.date) -> list[Query]:
+    token = _google_token(CONFIG / "gsc-service-account.json")
+    url = ("https://www.googleapis.com/webmasters/v3/sites/"
+           f"{urllib.parse.quote(SITE_PROPERTY, safe='')}/searchAnalytics/query")
+    rows: list[Query] = []
+    while True:
+        answer = _get_json(url, {"Authorization": f"Bearer {token}"}, {
+            "startDate": start.isoformat(), "endDate": end.isoformat(),
+            "dimensions": ["query", "page"], "rowLimit": 25000, "startRow": len(rows)})
+        batch = answer.get("rows", [])
+        rows += [Query(r["keys"][0], r["keys"][1], int(r["clicks"]), int(r["impressions"]),
+                       float(r["position"])) for r in batch]
+        if len(batch) < 25000:
+            return rows
+
+
+def fetch_goatcounter(start: dt.date, end: dt.date) -> dict[str, list[tuple[str, int]]]:
+    """Paths and events with their visit counts, and the referring domains."""
+    token = (CONFIG / "goatcounter-api-token.txt").read_text(encoding="utf-8").strip()
+    auth = {"Authorization": f"Bearer {token}"}
+    span = {"start": f"{start.isoformat()}T00:00:00Z", "end": f"{end.isoformat()}T23:59:59Z"}
+    hits: list[tuple[str, int]] = []
+    exclude: list[str] = []
+    while True:
+        params = urllib.parse.urlencode({**span, "limit": 100, "exclude_paths": ",".join(exclude)})
+        answer = _get_json(f"{GOATCOUNTER}/stats/hits?{params}", auth)
+        for hit in answer.get("hits", []):
+            hits.append((hit["path"] + (" [404]" if hit.get("title") == "404" else ""), int(hit["count"])))
+            exclude.append(str(hit["path_id"]))
+        if not answer.get("more"):
+            break
+    refs = _get_json(f"{GOATCOUNTER}/stats/toprefs?{urllib.parse.urlencode({**span, 'limit': 50})}", auth)
+    referrers = [(r.get("name") or "(direct)", int(r["count"])) for r in refs.get("stats", [])]
+    return {"hits": hits, "referrers": referrers}
+
+
+def fetch_github() -> dict:
+    token = github_token()
+    if not token:
+        raise SystemExit("GitHub traffic needs a token: GITHUB_TOKEN, or gh auth login")
+    auth = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}
+    api = f"https://api.github.com/repos/{REPOSITORY}"
+    releases = _get_json(f"{api}/releases?per_page=10", auth)
+    return {
+        "views": _get_json(f"{api}/traffic/views", auth),
+        "clones": _get_json(f"{api}/traffic/clones", auth),
+        "referrers": _get_json(f"{api}/traffic/popular/referrers", auth),
+        "paths": _get_json(f"{api}/traffic/popular/paths", auth),
+        "downloads": [(release["tag_name"], asset["name"], asset["download_count"])
+                      for release in releases for asset in release.get("assets", [])],
+    }
+
+
+# Reading the figures
+
+
+def searched_files(queries: list[Query], held: set[str], declared: set[str]) -> list[tuple[str, int, str]]:
+    """File names typed in searches, by impressions, with what the project
+    knows of each: held in the collection, declared by a profile only, or
+    unknown."""
+    seen: dict[str, int] = defaultdict(int)
+    shown: dict[str, str] = {}
+    for query in queries:
+        for token in FILE_TOKEN.findall(query.text):
+            key = token.casefold()
+            seen[key] += query.impressions
+            shown.setdefault(key, token)
+    state = {key: "held" if key in held else "declared, not held" if key in declared else "unknown"
+             for key in seen}
+    return sorted(((shown[key], seen[key], state[key]) for key in seen), key=lambda row: -row[1])
+
+
+def missed_clicks(queries: list[Query], least: int = 100) -> list[Query]:
+    """On the first page but rarely clicked: what the result says loses
+    the reader."""
+    return sorted((q for q in queries if q.position <= 10 and q.impressions >= least and q.ctr < 0.03),
+                  key=lambda q: -q.impressions)
+
+
+def near_first_page(queries: list[Query], least: int = 50) -> list[Query]:
+    return sorted((q for q in queries if 10 < q.position <= 20 and q.impressions >= least),
+                  key=lambda q: -q.impressions)
+
+
+def site_actions(hits: list[tuple[str, int]]) -> dict[str, list[tuple[str, int]]]:
+    """Events by kind ("search-none", "helpful", "download"...), pages apart."""
+    groups: dict[str, list[tuple[str, int]]] = defaultdict(list)
+    for path, count in hits:
+        if path.endswith(" [404]"):
+            groups["404"].append((path.removesuffix(" [404]"), count))
+        elif not path.startswith("/") and "/" in path:
+            kind, _, name = path.partition("/")
+            groups[kind].append((name, count))
+        else:
+            groups["pages"].append((path, count))
+    return {kind: sorted(rows, key=lambda row: -row[1]) for kind, rows in groups.items()}
+
+
+# The report
+
+
+def _table(header: tuple[str, ...], rows: list[tuple], limit: int = 25) -> list[str]:
+    if not rows:
+        return ["None.", ""]
+    lines = ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)]
+    lines += ["| " + " | ".join(str(cell) for cell in row) + " |" for row in rows[:limit]]
+    return [*lines, ""]
+
+
+def _page(url: str) -> str:
+    return url.removeprefix(SITE_PROPERTY.rstrip("/")) or "/"
+
+
+def report(start: dt.date, end: dt.date, queries: list[Query] | None, goat: dict | None,
+           github: dict | None, held: set[str], declared: set[str]) -> str:
+    out = [f"# Site insights, {start.isoformat()} to {end.isoformat()}", ""]
+    if queries is not None:
+        clicks = sum(q.clicks for q in queries)
+        shown = sum(q.impressions for q in queries)
+        out += ["## Google searches", "",
+                f"{clicks:,} clicks, {shown:,} impressions, {len({q.text for q in queries}):,} searches.", "",
+                "### On the first page, rarely clicked", "",
+                *_table(("Search", "Page", "Impressions", "CTR", "Position"),
+                        [(q.text, _page(q.page), f"{q.impressions:,}", f"{q.ctr:.1%}", f"{q.position:.1f}")
+                         for q in missed_clicks(queries)]),
+                "### Just past the first page", "",
+                *_table(("Search", "Page", "Impressions", "Position"),
+                        [(q.text, _page(q.page), f"{q.impressions:,}", f"{q.position:.1f}")
+                         for q in near_first_page(queries)]),
+                "### File names searched", "",
+                *_table(("File", "Impressions", "In the project"),
+                        [(f"`{name}`", f"{seen:,}", state)
+                         for name, seen, state in searched_files(queries, held, declared)], limit=40)]
+    if goat is not None:
+        actions = site_actions(goat["hits"])
+        named = {"search-none": "Site searches that found nothing", "404": "Broken addresses",
+                 "helpful": "Did this page help?", "get": "Pack pages opened",
+                 "download": "Release downloads", "file": "File downloads", "copy": "Copy buttons",
+                 "out": "Links leaving the site", "pages": "Pages"}
+        out += ["## On the site", ""]
+        for kind, title in named.items():
+            if kind in actions:
+                out += [f"### {title}", "", *_table(("Name", "Visits"), [(n, f"{c:,}") for n, c in actions[kind]])]
+        out += ["### Came from", "", *_table(("Site", "Visits"), [(n, f"{c:,}") for n, c in goat["referrers"]])]
+    if github is not None:
+        views, clones = github["views"], github["clones"]
+        out += ["## GitHub", "",
+                f"Last 14 days: {views.get('count', 0):,} views by {views.get('uniques', 0):,} visitors, "
+                f"{clones.get('count', 0):,} clones.", "",
+                "### Came from", "",
+                *_table(("Site", "Views", "Visitors"),
+                        [(r["referrer"], r["count"], r["uniques"]) for r in github["referrers"]]),
+                "### Most viewed", "",
+                *_table(("Path", "Views"), [(p["path"], p["count"]) for p in github["paths"]]),
+                "### Release downloads", "",
+                *_table(("Release", "File", "Downloads"),
+                        sorted(github["downloads"], key=lambda row: -row[2]), limit=40)]
+    return "\n".join(out).rstrip() + "\n"
+
+
+def collection_names(db: dict, profiles: dict[str, dict]) -> tuple[set[str], set[str]]:
+    """File names the collection holds, and those a profile declares."""
+    held = {name.casefold() for name in db.get("indexes", {}).get("by_name", {})}
+    declared: set[str] = set()
+    for profile in profiles.values():
+        for entry in profile.get("files") or []:
+            if isinstance(entry, dict):
+                for name in [entry.get("name"), *(entry.get("aliases") or [])]:
+                    if name:
+                        declared.add(Path(str(name)).name.casefold())
+    return held, declared
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False)
+    parser.add_argument("--days", type=int, default=28, help="days to read, ending 3 days ago (default 28)")
+    parser.add_argument("--skip", default="", help=f"sources to leave out, among {', '.join(SOURCES)}")
+    parser.add_argument("--output", type=Path, help="write the report here instead of stdout")
+    args = parser.parse_args()
+    skip = {name.strip() for name in args.skip.split(",") if name.strip()}
+    unknown = skip - set(SOURCES)
+    if unknown:
+        parser.error(f"unknown source(s): {', '.join(sorted(unknown))}")
+    # Search Console data settles two to three days after the fact.
+    end = dt.date.today() - dt.timedelta(days=3)
+    start = end - dt.timedelta(days=args.days - 1)
+    held, declared = collection_names(load_database(str(REPO_ROOT / "database.json")),
+                                      load_emulator_profiles(str(REPO_ROOT / "emulators")))
+    queries = None if "gsc" in skip else fetch_queries(start, end)
+    goat = None if "goatcounter" in skip else fetch_goatcounter(start, dt.date.today())
+    github = None if "github" in skip else fetch_github()
+    text = report(start, end, queries, goat, github, held, declared)
+    if args.output:
+        args.output.write_text(text, encoding="utf-8")
+        print(f"Wrote {args.output}", file=sys.stderr)
+    else:
+        sys.stdout.write(text)
+    return 0
+
+
+if __name__ == "__main__":
+    run_cli(main)

tests/test_site_insights.py

@@ -0,0 +1,67 @@
+"""The readings of scripts/site_insights.py, on synthetic figures."""
+from __future__ import annotations
+
+import datetime as dt
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+
+import site_insights as si  # noqa: E402
+
+PAGE = "https://abdess.github.io/retrobios/emulators/mgba/"
+
+
+class FileNamesInSearches(unittest.TestCase):
+    def test_each_name_carries_what_the_project_knows(self) -> None:
+        queries = [si.Query("gba_bios.bin download", PAGE, 1, 300, 8.0),
+                   si.Query("where to put GBA_BIOS.BIN", PAGE, 0, 50, 9.0),
+                   si.Query("dsp1b.rom snes", PAGE, 0, 40, 9.0),
+                   si.Query("mystery.pup", PAGE, 0, 10, 30.0),
+                   si.Query("gba bios", PAGE, 5, 900, 7.0)]
+        rows = si.searched_files(queries, held={"gba_bios.bin"}, declared={"gba_bios.bin", "dsp1b.rom"})
+        self.assertEqual(rows, [("gba_bios.bin", 350, "held"), ("dsp1b.rom", 40, "declared, not held"),
+                                ("mystery.pup", 10, "unknown")])
+
+
+class SearchGaps(unittest.TestCase):
+    def test_first_page_rarely_clicked(self) -> None:
+        low = si.Query("swanstation", PAGE, 3, 540, 5.6)
+        fine = si.Query("nethersx2 bios", PAGE, 54, 484, 4.6)
+        far = si.Query("xbox bios", PAGE, 0, 900, 10.7)
+        self.assertEqual(si.missed_clicks([low, fine, far]), [low])
+        self.assertEqual(si.near_first_page([low, fine, far]), [far])
+
+
+class SiteActions(unittest.TestCase):
+    def test_events_are_grouped_by_kind_and_pages_apart(self) -> None:
+        hits = [("/emulators/mgba/", 40), ("search-none/kenji nx", 3), ("search-none/dsp1b.rom", 7),
+                ("helpful/no", 2), ("/old/link/ [404]", 5), ("download/Batocera_43.1_BIOS_Pack.zip", 9)]
+        actions = si.site_actions(hits)
+        self.assertEqual(actions["search-none"], [("dsp1b.rom", 7), ("kenji nx", 3)])
+        self.assertEqual(actions["404"], [("/old/link/", 5)])
+        self.assertEqual(actions["pages"], [("/emulators/mgba/", 40)])
+        self.assertEqual(actions["helpful"], [("no", 2)])
+
+
+class TheReport(unittest.TestCase):
+    def test_a_skipped_source_leaves_no_section(self) -> None:
+        text = si.report(dt.date(2026, 10, 1), dt.date(2026, 10, 28),
+                         [si.Query("gba_bios.bin", PAGE, 1, 300, 8.0)], None, None,
+                         held={"gba_bios.bin"}, declared=set())
+        self.assertIn("## Google searches", text)
+        self.assertIn("| gba_bios.bin | /emulators/mgba/ | 300 | 0.3% | 8.0 |", text)
+        self.assertNotIn("## On the site", text)
+        self.assertNotIn("## GitHub", text)
+
+    def test_names_come_from_profiles_and_the_collection(self) -> None:
+        db = {"indexes": {"by_name": {"SCPH5501.BIN": ["x"]}}}
+        profiles = {"p": {"files": [{"name": "bios/dsp1b.rom", "aliases": ["DSP1B.bin"]}, "str"]}}
+        held, declared = si.collection_names(db, profiles)
+        self.assertEqual(held, {"scph5501.bin"})
+        self.assertEqual(declared, {"dsp1b.rom", "dsp1b.bin"})
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/testing-guide.md

@@ -65,7 +65,7 @@ Every module in `tests/`, by the area it guards.
 | Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` |
 | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
-| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_insights`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_privacy`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` |
 | Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` |
 
 ## Test architecture

wiki/tools.md

@@ -1353,6 +1353,7 @@ before a release (wiki/release-process.md). Its lines stay in
 | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |
 | `generate_site.py` | Generate the site (this documentation) into the Starlight project in `website/` |
 | `sitesources.py` | Record which cited source paths each pinned upstream tree holds, for the site's source links (`--fetch` reads a tree no cache holds) |
+| `site_insights.py` | What readers look for and do not find, from totals only: searches that show the site but are rarely clicked, file names searched and whether the collection holds them, site searches that found nothing, broken addresses, "Did this page help?" answers, GitHub traffic and release downloads (`--days`, `--skip gsc,goatcounter,github`, `--output`). Reads Search Console, GoatCounter and GitHub with credentials kept outside the repository. [What the site counts](privacy.md) |
 | `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets |
 | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held |
 | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch <revision>` reads FBNeo at that tag, branch or commit |