diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..e974522 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,24 @@ +User-Agent: GPTBot +User-Agent: ChatGPT-User +User-Agent: ClaudeBot +User-Agent: anthropic-ai +User-Agent: CCBot +User-Agent: FacebookBot +User-Agent: Twitterbot +User-Agent: PerplexityBot +User-Agent: OmigiliBot +User-Agent: AISearchBot +User-Agent: mlbot +User-Agent: PetalBot +User-Agent: Amazonbot +User-Agent: DuckAssistBot +User-Agent: OAI-SearchBot +User-Agent: OAI-SearchBot +User-Agent: omgili +User-Agent: omgilibot +User-Agent: PerplexityBot +User-Agent: Webzio-Extended +User-Agent: Meta-ExternalAgent +User-Agent: Meta-ExternalFetcher +User-Agent: cohere-ai +Disallow: / \ No newline at end of file