diff --git a/docs_assets/overrides/main.html b/docs_assets/overrides/main.html
index 3bce5e10..4348d78c 100644
--- a/docs_assets/overrides/main.html
+++ b/docs_assets/overrides/main.html
@@ -23,6 +23,10 @@
{% if page_url %}{% endif %}
{% if preview_image %}{% endif %}
+ {# The sitemap, named from every page: a project site under /retrobios/
+ cannot serve the host-root robots.txt that would announce it. #}
+ {% if config.site_url %}{% endif %}
+
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 0e4e1b25..6183bcc0 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -4226,12 +4226,10 @@ def _generate(args: argparse.Namespace) -> None:
js_dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(js_src, js_dest)
- # mkdocs writes sitemap.xml but no robots.txt, so nothing points a crawler
- # at it. Everything published here is meant to be indexed.
- write_if_changed(
- str(docs / "robots.txt"),
- "User-agent: *\nAllow: /\n\nSitemap: https://abdess.github.io/retrobios/sitemap.xml\n",
- )
+ # No robots.txt: a crawler reads it only at the host root (RFC 9309), and
+ # this project site lives under /retrobios/. Every page's head points at
+ # the sitemap instead (overrides/main.html).
+ (docs / "robots.txt").unlink(missing_ok=True)
# Copy branding assets
images_dest = docs / "assets" / "images"
diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py
index fc2fb6cb..0ef63412 100644
--- a/tests/test_site_validation.py
+++ b/tests/test_site_validation.py
@@ -217,5 +217,20 @@ class TheTestingGuideNamesEveryModule(unittest.TestCase):
self.assertEqual(missing, [])
+
+class TheSitemapIsAnnouncedWhereCrawlersLook(unittest.TestCase):
+ """robots.txt was published as /retrobios/robots.txt, which no crawler
+ reads (RFC 9309 reads the host root): the sitemap it announced was
+ never found that way."""
+
+ def test_the_head_names_it_and_no_robots_is_written(self):
+ root = Path(__file__).resolve().parents[1]
+ override = (root / "docs_assets" / "overrides" / "main.html").read_text()
+ self.assertIn('rel="sitemap"', override)
+ source = (root / "scripts" / "generate_site.py").read_text()
+ self.assertNotIn('"robots.txt"),\n "User-agent', source)
+ self.assertNotIn("Sitemap: https://", source)
+
+
if __name__ == "__main__":
unittest.main()