diff --git a/public/robots.txt b/public/robots.txt index 26ccaa30000069..f7de331f64284f 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -20,6 +20,12 @@ User-agent: * Disallow: /media_proxy/ Disallow: /interact/ +# AI Data Scraper +# https://darkvisitors.com/agents/ai2bot + +User-agent: AI2Bot +Disallow: / + # AI Search Crawler # https://darkvisitors.com/agents/amazonbot @@ -86,6 +92,12 @@ Disallow: / User-agent: Diffbot Disallow: / +# AI Assistant +# https://darkvisitors.com/agents/duckassistbot + +User-agent: DuckAssistBot +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/facebookbot @@ -104,6 +116,12 @@ Disallow: / User-agent: GPTBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/kangaroo-bot + +User-agent: Kangaroo Bot +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/meta-externalagent