Merge pull request #66 from NeuromatchAcademy/robots.txt-update

Update robots.txt
NeuromatchAcademy · Nov 3, 2024 · c3a1df6 · c3a1df6
2 parents 00d9cb9 + cb49e85
commit c3a1df6
Showing 1 changed file with 18 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -20,6 +20,12 @@ User-agent: *
 Disallow: /media_proxy/
 Disallow: /interact/
 
+# AI Data Scraper
+# https://darkvisitors.com/agents/ai2bot
+
+User-agent: AI2Bot
+Disallow: /
+
 # AI Search Crawler
 # https://darkvisitors.com/agents/amazonbot
 
@@ -86,6 +92,12 @@ Disallow: /
 User-agent: Diffbot
 Disallow: /
 
+# AI Assistant
+# https://darkvisitors.com/agents/duckassistbot
+
+User-agent: DuckAssistBot
+Disallow: /
+
 # AI Data Scraper
 # https://darkvisitors.com/agents/facebookbot
 
@@ -104,6 +116,12 @@ Disallow: /
 User-agent: GPTBot
 Disallow: /
 
+# AI Data Scraper
+# https://darkvisitors.com/agents/kangaroo-bot
+
+User-agent: Kangaroo Bot
+Disallow: /
+
 # AI Data Scraper
 # https://darkvisitors.com/agents/meta-externalagent