robots: also disallow %5E (urlencoded caret)

rust-lang · Jan 8, 2025 · ff58850 · ff58850
1 parent c1be79c
commit ff58850
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/static/robots.txt b/static/robots.txt
@@ -4,4 +4,7 @@ Sitemap: https://docs.rs/sitemap.xml
 # https://docs.rs/about/redirections
 User-Agent: *
 Disallow: */^
+# %5E is '^', URL-encoded. Based on the Search Console, Google
+# may be encoding '^' before checking against robots.txt.
+Disallow: */%5E
 Disallow: */~