mirror of
https://github.com/standardebooks/web.git
synced 2025-07-13 10:02:02 -04:00
Update robots.txt to disallow downloads by SEO bots
This commit is contained in:
parent
475c437126
commit
f6aef54767
1 changed files with 8 additions and 0 deletions
|
@ -1,4 +1,12 @@
|
|||
Sitemap: https://standardebooks.org/sitemap
|
||||
|
||||
# Badly-behaved bots
|
||||
User-agent: *
|
||||
Disallow: /honeypot
|
||||
|
||||
# SEO crawlers
|
||||
User-agent: SemrushBot
|
||||
User-agent: DotBot
|
||||
User-agent: AhrefsBot
|
||||
User-agent: SEOkicks
|
||||
Disallow: /ebooks/*/downloads/*
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue