From f6aef5476765f6d8b971b9a3e9d4b11b4e822e5e Mon Sep 17 00:00:00 2001
From: Alex Cabal <alex@alexcabal.com>
Date: Thu, 22 May 2025 13:24:57 -0500
Subject: [PATCH] Update robots.txt to disallow downloads by SEO bots

---
 www/robots.txt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/www/robots.txt b/www/robots.txt
index 48211b7f..665c8d6b 100644
--- a/www/robots.txt
+++ b/www/robots.txt
@@ -1,4 +1,12 @@
 Sitemap: https://standardebooks.org/sitemap
 
+# Badly-behaved bots
 User-agent: *
 Disallow: /honeypot
+
+# SEO crawlers
+User-agent: SemrushBot
+User-agent: DotBot
+User-agent: AhrefsBot
+User-agent: SEOkicks
+Disallow: /ebooks/*/downloads/*