From 2268fa9e5b5c871a95b723fcf686a8f5857b3894 Mon Sep 17 00:00:00 2001
From: hobgoblina <20880695+hobgoblina@users.noreply.github.com>
Date: Thu, 11 Apr 2024 05:23:02 +0000
Subject: [PATCH] Update robots.txt

---
 public/robots.txt | 105 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 101 insertions(+), 4 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index 6672eeba1f22d7..613c48e6c4c673 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,8 +1,105 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-
-User-agent: GPTBot
-Disallow: /
+#         .__---~~~(~~-_.
+#      _-'  ) -~~- ) _-" )_
+#     (  ( `-,_..`.,_--_ '_,)_
+#    (  -_)  ( -_-~  -_ `,    )
+#    (_ -_ _-~-__-~`, ,' )__-'))--___--~~~--__--~~--___--__..
+#    _ ~`_-'( (____;--==,,_))))--___--~~~--__--~~--__----~~~'`=__-~+_-_.
+#   (@) (@) `````      `-_(())_-~
+#
+#                            ,---.       .=-.-..-._          ,-,--.  
+#       _..---.   .-.,.---.  .--.'  \     /==/_ /==/ \  .-._ ,-.'-  _\ 
+#     .' .'.-. \ /==/  `   \ \==\-/\ \   |==|, ||==|, \/ /, /==/_ ,_.' 
+#    /==/- '=' /|==|-, .=., |/==/-|_\ |  |==|  ||==|-  \|  |\==\  \    
+#    |==|-,   ' |==|   '='  /\==\,   - \ |==|- ||==| ,  | -| \==\ -\   
+#    |==|  .=. \|==|- ,   .' /==/ -   ,| |==| ,||==| -   _ | _\==\ ,\  
+#    /==/- '=' ,|==|_  . ,'./==/-  /\ - \|==|- ||==|  /\ , |/==/\/ _ | 
+#   |==|   -   //==/  /\ ,  )==\ _.\=\.-'/==/. //==/, | |- |\==\ - , / 
+#   `-._`.___,' `--`-`--`--' `--`        `--`-` `--`./  `--` `--`---'
 
 User-agent: *
 Disallow: /media_proxy/
 Disallow: /interact/
+
+# AI Search Crawler
+# https://darkvisitors.com/agents/amazonbot
+
+User-agent: Amazonbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# AI Search Crawler
+# https://darkvisitors.com/agents/applebot
+
+User-agent: Applebot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
+User-agent: Bytespider
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Assistant
+# https://darkvisitors.com/agents/chatgpt-user
+
+User-agent: ChatGPT-User
+Disallow: /
+
+# AI Assistant
+# https://darkvisitors.com/agents/cohere-ai
+
+User-agent: cohere-ai
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/diffbot
+
+User-agent: Diffbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
+User-agent: GPTBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /
+
+# AI Search Crawler
+# https://darkvisitors.com/agents/perplexitybot
+
+User-agent: PerplexityBot
+Disallow: /
+
+# AI Search Crawler
+# https://darkvisitors.com/agents/youbot
+
+User-agent: YouBot
+Disallow: /
\ No newline at end of file