diff --git a/docs_assets/overrides/main.html b/docs_assets/overrides/main.html index 3bce5e10..4348d78c 100644 --- a/docs_assets/overrides/main.html +++ b/docs_assets/overrides/main.html @@ -23,6 +23,10 @@ {% if page_url %}{% endif %} {% if preview_image %}{% endif %} + {# The sitemap, named from every page: a project site under /retrobios/ + cannot serve the host-root robots.txt that would announce it. #} + {% if config.site_url %}{% endif %} + diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 0e4e1b25..6183bcc0 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4226,12 +4226,10 @@ def _generate(args: argparse.Namespace) -> None: js_dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(js_src, js_dest) - # mkdocs writes sitemap.xml but no robots.txt, so nothing points a crawler - # at it. Everything published here is meant to be indexed. - write_if_changed( - str(docs / "robots.txt"), - "User-agent: *\nAllow: /\n\nSitemap: https://abdess.github.io/retrobios/sitemap.xml\n", - ) + # No robots.txt: a crawler reads it only at the host root (RFC 9309), and + # this project site lives under /retrobios/. Every page's head points at + # the sitemap instead (overrides/main.html). + (docs / "robots.txt").unlink(missing_ok=True) # Copy branding assets images_dest = docs / "assets" / "images" diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py index fc2fb6cb..0ef63412 100644 --- a/tests/test_site_validation.py +++ b/tests/test_site_validation.py @@ -217,5 +217,20 @@ class TheTestingGuideNamesEveryModule(unittest.TestCase): self.assertEqual(missing, []) + +class TheSitemapIsAnnouncedWhereCrawlersLook(unittest.TestCase): + """robots.txt was published as /retrobios/robots.txt, which no crawler + reads (RFC 9309 reads the host root): the sitemap it announced was + never found that way.""" + + def test_the_head_names_it_and_no_robots_is_written(self): + root = Path(__file__).resolve().parents[1] + override = (root / "docs_assets" / "overrides" / "main.html").read_text() + self.assertIn('rel="sitemap"', override) + source = (root / "scripts" / "generate_site.py").read_text() + self.assertNotIn('"robots.txt"),\n "User-agent', source) + self.assertNotIn("Sitemap: https://", source) + + if __name__ == "__main__": unittest.main()