diff --git a/www/robots.txt b/www/robots.txt index 48211b7f..665c8d6b 100644 --- a/www/robots.txt +++ b/www/robots.txt @@ -1,4 +1,12 @@ Sitemap: https://standardebooks.org/sitemap +# Badly-behaved bots User-agent: * Disallow: /honeypot + +# SEO crawlers +User-agent: SemrushBot +User-agent: DotBot +User-agent: AhrefsBot +User-agent: SEOkicks +Disallow: /ebooks/*/downloads/*