fix: announce the sitemap from each page

docs_assets/overrides/main.html

@@ -23,6 +23,10 @@
   {% if page_url %}<meta property="og:url" content="{{ page_url }}">{% endif %}
   {% if preview_image %}<meta property="og:image" content="{{ preview_image }}">{% endif %}
 
+  {# The sitemap, named from every page: a project site under /retrobios/
+     cannot serve the host-root robots.txt that would announce it. #}
+  {% if config.site_url %}<link rel="sitemap" type="application/xml" title="Sitemap" href="{{ config.site_url }}sitemap.xml">{% endif %}
+
   <meta name="twitter:card" content="summary">
   <meta name="twitter:title" content="{{ page_title }}">
   <meta name="twitter:description" content="{{ page_description }}">

scripts/generate_site.py

@@ -4226,12 +4226,10 @@ def _generate(args: argparse.Namespace) -> None:
         js_dest.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(js_src, js_dest)
 
-    # mkdocs writes sitemap.xml but no robots.txt, so nothing points a crawler
-    # at it. Everything published here is meant to be indexed.
-    write_if_changed(
-        str(docs / "robots.txt"),
-        "User-agent: *\nAllow: /\n\nSitemap: https://abdess.github.io/retrobios/sitemap.xml\n",
-    )
+    # No robots.txt: a crawler reads it only at the host root (RFC 9309), and
+    # this project site lives under /retrobios/. Every page's head points at
+    # the sitemap instead (overrides/main.html).
+    (docs / "robots.txt").unlink(missing_ok=True)
 
     # Copy branding assets
     images_dest = docs / "assets" / "images"

tests/test_site_validation.py

@@ -217,5 +217,20 @@ class TheTestingGuideNamesEveryModule(unittest.TestCase):
         self.assertEqual(missing, [])
 
 
+
+class TheSitemapIsAnnouncedWhereCrawlersLook(unittest.TestCase):
+    """robots.txt was published as /retrobios/robots.txt, which no crawler
+    reads (RFC 9309 reads the host root): the sitemap it announced was
+    never found that way."""
+
+    def test_the_head_names_it_and_no_robots_is_written(self):
+        root = Path(__file__).resolve().parents[1]
+        override = (root / "docs_assets" / "overrides" / "main.html").read_text()
+        self.assertIn('rel="sitemap"', override)
+        source = (root / "scripts" / "generate_site.py").read_text()
+        self.assertNotIn('"robots.txt"),\n        "User-agent', source)
+        self.assertNotIn("Sitemap: https://", source)
+
+
 if __name__ == "__main__":
     unittest.main()