Skip to content

Commit

Permalink
Merge branch 'dev' into merge-upstream
Browse files Browse the repository at this point in the history
  • Loading branch information
sneakers-the-rat committed Jun 2, 2024
2 parents 5eb0faa + 23c90f5 commit f423258
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 4 deletions.
34 changes: 34 additions & 0 deletions .github/workflows/update-robots-txt.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: Update robots.txt

on:
schedule:
- cron: '0 0 * * 0'
workflow_dispatch:

jobs:
update-robots:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Update robot.txt
id: update
env:
API_KEY: ${{ secrets.ROBOTS }}
run: |
cp robots-base.txt public/robots.txt
curl --location 'https://api.darkvisitors.com/robots-txts' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $API_KEY" \
--data '{ "agent_types": [ "AI Data Scraper", "AI Assistant", "AI Search Crawler", "Undocumented AI Agent" ], "disallow": "/" }' >> public/robots.txt
- name: Create pull request
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
branch: robots.txt-update
title: "Update robots.txt"
commit-message: "Update robots.txt"
labels: 'robots.txt'
add-paths: public/robots.txt
reviewers: hobgoblina,mannazsci,sneakers-the-rat
body: This PR was generated by the `Update robots.txt` action and contains updates to our robots.txt file, pulled from [Dark Visitors](https://darkvisitors.com/).
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
.status {
&.collapsed {
.status__content {
height: unset;
max-height: 6em;
padding-top: unset;

&::after {
top: unset;
height: 40%;
}
}
}

&--in-thread {
&.collapsed {
& > button {
margin-inline-start: 46px + 10px;
}
}
}
}
117 changes: 113 additions & 4 deletions public/robots.txt
Original file line number Diff line number Diff line change
@@ -1,8 +1,117 @@
# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

User-agent: GPTBot
Disallow: /
# .__---~~~(~~-_.
# _-' ) -~~- ) _-" )_
# ( ( `-,_..`.,_--_ '_,)_
# ( -_) ( -_-~ -_ `, )
# (_ -_ _-~-__-~`, ,' )__-'))--___--~~~--__--~~--___--__..
# _ ~`_-'( (____;--==,,_))))--___--~~~--__--~~--__----~~~'`=__-~+_-_.
# (@) (@) ````` `-_(())_-~
#
# ,---. .=-.-..-._ ,-,--.
# _..---. .-.,.---. .--.' \ /==/_ /==/ \ .-._ ,-.'- _\
# .' .'.-. \ /==/ ` \ \==\-/\ \ |==|, ||==|, \/ /, /==/_ ,_.'
# /==/- '=' /|==|-, .=., |/==/-|_\ | |==| ||==|- \| |\==\ \
# |==|-, ' |==| '=' /\==\, - \ |==|- ||==| , | -| \==\ -\
# |==| .=. \|==|- , .' /==/ - ,| |==| ,||==| - _ | _\==\ ,\
# /==/- '=' ,|==|_ . ,'./==/- /\ - \|==|- ||==| /\ , |/==/\/ _ |
# |==| - //==/ /\ , )==\ _.\=\.-'/==/. //==/, | |- |\==\ - , /
# `-._`.___,' `--`-`--`--' `--` `--`-` `--`./ `--` `--`---'

User-agent: *
Disallow: /media_proxy/
Disallow: /interact/

# AI Search Crawler
# https://darkvisitors.com/agents/amazonbot

User-agent: Amazonbot
Disallow: /

# Undocumented AI Agent
# https://darkvisitors.com/agents/anthropic-ai

User-agent: anthropic-ai
Disallow: /

# AI Search Crawler
# https://darkvisitors.com/agents/applebot

User-agent: Applebot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/bytespider

User-agent: Bytespider
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/ccbot

User-agent: CCBot
Disallow: /

# AI Assistant
# https://darkvisitors.com/agents/chatgpt-user

User-agent: ChatGPT-User
Disallow: /

# Undocumented AI Agent
# https://darkvisitors.com/agents/claude-web

User-agent: Claude-Web
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/claudebot

User-agent: ClaudeBot
Disallow: /

# Undocumented AI Agent
# https://darkvisitors.com/agents/cohere-ai

User-agent: cohere-ai
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/diffbot

User-agent: Diffbot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/facebookbot

User-agent: FacebookBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/google-extended

User-agent: Google-Extended
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/gptbot

User-agent: GPTBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/omgili

User-agent: omgili
Disallow: /

# AI Search Crawler
# https://darkvisitors.com/agents/perplexitybot

User-agent: PerplexityBot
Disallow: /

# AI Search Crawler
# https://darkvisitors.com/agents/youbot

User-agent: YouBot
Disallow: /
22 changes: 22 additions & 0 deletions robots-base.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# .__---~~~(~~-_.
# _-' ) -~~- ) _-" )_
# ( ( `-,_..`.,_--_ '_,)_
# ( -_) ( -_-~ -_ `, )
# (_ -_ _-~-__-~`, ,' )__-'))--___--~~~--__--~~--___--__..
# _ ~`_-'( (____;--==,,_))))--___--~~~--__--~~--__----~~~'`=__-~+_-_.
# (@) (@) ````` `-_(())_-~
#
# ,---. .=-.-..-._ ,-,--.
# _..---. .-.,.---. .--.' \ /==/_ /==/ \ .-._ ,-.'- _\
# .' .'.-. \ /==/ ` \ \==\-/\ \ |==|, ||==|, \/ /, /==/_ ,_.'
# /==/- '=' /|==|-, .=., |/==/-|_\ | |==| ||==|- \| |\==\ \
# |==|-, ' |==| '=' /\==\, - \ |==|- ||==| , | -| \==\ -\
# |==| .=. \|==|- , .' /==/ - ,| |==| ,||==| - _ | _\==\ ,\
# /==/- '=' ,|==|_ . ,'./==/- /\ - \|==|- ||==| /\ , |/==/\/ _ |
# |==| - //==/ /\ , )==\ _.\=\.-'/==/. //==/, | |- |\==\ - , /
# `-._`.___,' `--`-`--`--' `--` `--`-` `--`./ `--` `--`---'

User-agent: *
Disallow: /media_proxy/
Disallow: /interact/

0 comments on commit f423258

Please sign in to comment.