diff --git a/public/robots.txt b/public/robots.txt index 7f56cdbd..4eec88da 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,6 +1,7 @@ # Stop crawlers running searches (via facet links) User-agent: * Disallow: /catalog +Disallow: */pdf_pages/* # but do give them some lovely resource-sync sitemaps Sitemap: https://bl.iro.bl.uk/resourcelist Sitemap: https://mola.iro.bl.uk/resourcelist