fix: pair each source repository with its own pin

A profile whose builds live in separate repositories keys source,
upstream and source_commit by build mode. The site flattened the URLs
but read a single scalar revision, so a libretro fork could be pinned
to the standalone commit, and binding the object form into SQLite
failed outright once ymir adopted it. URL and revision are now read as
pairs.

The site also published a sitemap nothing pointed at, so robots.txt is
generated alongside it.

scripts/generate_site.py

@@ -30,8 +30,8 @@ from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import (
-    GAME_DATA_TOPS,
     compute_composition,
+    GAME_DATA_TOPS,
     list_registered_platforms,
     load_database,
     load_emulator_profiles,
@@ -40,6 +40,7 @@ from common import (
     require_yaml,
     unique_emulator_profiles,
     write_if_changed,
+    yaml_load,
 )
 
 yaml = require_yaml()
@@ -72,22 +73,47 @@ def _forge_sources(profile: dict, label: str = "") -> list[tuple[upstream.Repo,
 
     for field in ("source", "upstream"):
         raw = profile.get(field, "")
-        values: list[str] = []
-        if isinstance(raw, dict):
-            if label and isinstance(raw.get(label), str):
-                values.append(raw[label])
-            values.extend(str(value) for value in raw.values() if value)
-        elif raw:
-            values.append(str(raw))
+        raw_pin = profile.get(f"{field}_commit") or ""
+        source_pin = profile.get("source_commit") or ""
 
-        for value in values:
+        # A profile whose builds live in separate repositories keys both the
+        # URL and the revision by build mode. The two have to be read as
+        # pairs: pinning a libretro fork to the standalone revision would
+        # produce a permalink into the wrong tree.
+        pairs: list[tuple[str, str]] = []
+        if isinstance(raw, dict):
+            ordered = ([label] if label and label in raw else []) + [
+                key for key in raw if key != label
+            ]
+            for key in ordered:
+                value = raw.get(key)
+                if not value:
+                    continue
+                pin = raw_pin.get(key, "") if isinstance(raw_pin, dict) else raw_pin
+                fallback = (
+                    source_pin.get(key, "")
+                    if isinstance(source_pin, dict)
+                    else source_pin
+                )
+                pairs.append((str(value), str(pin or ""), str(fallback or "")))
+        elif raw:
+            pin = raw_pin.get(label, "") if isinstance(raw_pin, dict) and label else (
+                "" if isinstance(raw_pin, dict) else raw_pin
+            )
+            fallback = (
+                source_pin.get(label, "")
+                if isinstance(source_pin, dict) and label
+                else ("" if isinstance(source_pin, dict) else source_pin)
+            )
+            pairs.append((str(raw), str(pin or ""), str(fallback or "")))
+
+        for value, pin, fallback in pairs:
             repo = upstream.parse_repo(value)
             if repo is None:
                 continue
             repo_key = (repo.host, repo.slug)
-            pin = str(profile.get(f"{field}_commit") or "")
             if field == "upstream" and not pin and repo_key in source_repos:
-                pin = str(profile.get("source_commit") or "")
+                pin = fallback
             if not pin:
                 continue
             key = (repo.host, repo.slug, pin)
@@ -455,14 +481,6 @@ def _pct(n: int, total: int) -> str:
     return f"{n / total * 100:.1f}%"
 
 
-def _status_icon(pct: float) -> str:
-    if pct >= 100:
-        return "OK"
-    if pct >= 95:
-        return "~OK"
-    return "partial"
-
-
 # Home page
 
 
@@ -891,7 +909,7 @@ def _cross_reference_export_rows(coverages: dict, profiles: dict) -> list[dict]:
                         "type": profile.get("type", ""),
                         "source": _json_text(profile.get("source")),
                         "upstream": _json_text(profile.get("upstream")),
-                        "profiled_commit": profile.get("source_commit", ""),
+                        "profiled_commit": _json_text(profile.get("source_commit", "")),
                         "file_count": len(profile.get("files", []) or []),
                     })
     return rows
@@ -1018,7 +1036,11 @@ def _write_sqlite_export(
                     profile.get("core_classification", ""),
                     _json_text(profile.get("source")),
                     _json_text(profile.get("upstream")),
-                    profile.get("source_commit", ""), _json_text(profile),
+                    # source_commit is a string or, when the builds live in
+                    # separate repositories, an object keyed by build mode.
+                    # SQLite cannot bind the object form.
+                    _json_text(profile.get("source_commit", "")),
+                    _json_text(profile),
                 ),
             )
             for system in sorted(set(profile.get("systems", []) or [])):
@@ -1408,9 +1430,7 @@ def decorate_markdown_pages(docs: Path) -> None:
 
 
 def generate_platform_index(coverages: dict, registry: dict | None = None) -> str:
-    total_files = sum(c["total"] for c in coverages.values())
     total_present = sum(c["present"] for c in coverages.values())
-    total_verified = sum(c["verified"] for c in coverages.values())
 
     lines = [
         f"# Platforms - {SITE_NAME}",
@@ -1492,13 +1512,6 @@ def generate_platform_page(
     base_dest = config.get("base_destination", "")
 
     pct_val = cov["present"] / cov["total"] * 100 if cov["total"] else 0
-    coverage_badge = (
-        "rb-badge-success"
-        if pct_val >= 95
-        else "rb-badge-warning"
-        if pct_val >= 70
-        else "rb-badge-danger"
-    )
     mode_badge = (
         "rb-badge-success" if mode in ("md5", "sha1") else "rb-badge-info"
     )
@@ -3788,6 +3801,13 @@ def main():
         js_dest.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(js_src, js_dest)
 
+    # mkdocs writes sitemap.xml but no robots.txt, so nothing points a crawler
+    # at it. Everything published here is meant to be indexed.
+    write_if_changed(
+        str(docs / "robots.txt"),
+        "User-agent: *\nAllow: /\n\nSitemap: https://abdess.github.io/retrobios/sitemap.xml\n",
+    )
+
     # Copy branding assets
     images_dest = docs / "assets" / "images"
     images_dest.mkdir(parents=True, exist_ok=True)
@@ -3801,7 +3821,7 @@ def main():
     registry = {}
     if registry_path.exists():
         with open(registry_path) as f:
-            registry = (yaml.safe_load(f) or {}).get("platforms", {})
+            registry = (yaml_load(f) or {}).get("platforms", {})
 
     platform_names = list_registered_platforms(
         args.platforms_dir, include_archived=True

tests/test_site_exports.py

@@ -16,6 +16,7 @@ sys.path.insert(0, str(ROOT / "scripts"))
 from generate_site import (  # noqa: E402
     _admonition_body,
     _browser_title,
+    _forge_sources,
     _source_ref_markdown,
     decorate_markdown_pages,
     generate_data_exports,
@@ -203,5 +204,58 @@ class SiteReferenceContracts(unittest.TestCase):
         self.assertIn('"@type":"TechArticle"', content)
 
 
+class PerModeSourcePins(unittest.TestCase):
+    """A profile whose builds live in separate repositories.
+
+    `source`, `upstream` and `source_commit` may each be keyed by build mode.
+    Reading the URL from one mode and the revision from another produces a
+    permalink into a tree that never held that line, and binding the object
+    form straight into SQLite fails outright.
+    """
+
+    PROFILE = {
+        "emulator": "twobuilds",
+        "source": {
+            "standalone": "https://github.com/vendor/emu",
+            "libretro": "https://github.com/porter/emu-libretro",
+        },
+        "upstream": "https://github.com/vendor/emu",
+        "source_commit": {"standalone": "a" * 40, "libretro": "b" * 40},
+    }
+
+    def test_each_repository_keeps_its_own_revision(self):
+        pairs = {
+            (repo.slug, pin) for repo, pin in _forge_sources(self.PROFILE)
+        }
+        self.assertIn(("vendor/emu", "a" * 40), pairs)
+        self.assertIn(("porter/emu-libretro", "b" * 40), pairs)
+        # The libretro fork must never be pinned to the standalone revision.
+        self.assertNotIn(("porter/emu-libretro", "a" * 40), pairs)
+        self.assertNotIn(("vendor/emu", "b" * 40), pairs)
+
+    def test_a_build_mode_label_is_resolved_first(self):
+        repo, pin = _forge_sources(self.PROFILE, "libretro")[0]
+        self.assertEqual((repo.slug, pin), ("porter/emu-libretro", "b" * 40))
+
+    def test_the_plain_string_form_is_unchanged(self):
+        profile = {
+            "source": "https://github.com/vendor/emu",
+            "source_commit": "c" * 40,
+        }
+        self.assertEqual(
+            [(repo.slug, pin) for repo, pin in _forge_sources(profile)],
+            [("vendor/emu", "c" * 40)],
+        )
+
+    def test_sqlite_export_accepts_a_per_mode_commit(self):
+        from generate_site import _json_text
+
+        self.assertEqual(
+            _json_text(self.PROFILE["source_commit"]),
+            '{"libretro": "' + "b" * 40 + '", "standalone": "' + "a" * 40 + '"}',
+        )
+        self.assertEqual(_json_text("c" * 40), "c" * 40)
+
+
 if __name__ == "__main__":
     unittest.main()