diff --git a/.github/workflows/update-robots-txt.yml b/.github/workflows/update-robots-txt.yml new file mode 100644 index 00000000000000..b2a7928c22466b --- /dev/null +++ b/.github/workflows/update-robots-txt.yml @@ -0,0 +1,34 @@ +name: Update robots.txt + +on: + schedule: + - cron: '0 0 * * 0' + workflow_dispatch: + +jobs: + update-robots: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Update robot.txt + id: update + env: + API_KEY: ${{ secrets.ROBOTS }} + run: | + cp robots-base.txt public/robots.txt + curl --location 'https://api.darkvisitors.com/robots-txts' \ + --header 'Content-Type: application/json' \ + --header "Authorization: Bearer $API_KEY" \ + --data '{ "agent_types": [ "AI Data Scraper", "AI Assistant", "AI Search Crawler" ], "disallow": "/" }' >> public/robots.txt + - name: Create pull request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: robots.txt-update + title: "Update robots.txt" + commit-message: "Update robots.txt" + labels: 'robots.txt' + add-paths: public/robots.txt + reviewers: hobgoblina,mannazsci,sneakers-the-rat + body: This PR was generated by the `Update robots.txt` action and contains updates to our robots.txt file, pulled from [Dark Visitors](https://darkvisitors.com/). diff --git a/robots-base.txt b/robots-base.txt new file mode 100644 index 00000000000000..53a70c15660140 --- /dev/null +++ b/robots-base.txt @@ -0,0 +1,22 @@ +# .__---~~~(~~-_. +# _-' ) -~~- ) _-" )_ +# ( ( `-,_..`.,_--_ '_,)_ +# ( -_) ( -_-~ -_ `, ) +# (_ -_ _-~-__-~`, ,' )__-'))--___--~~~--__--~~--___--__.. +# _ ~`_-'( (____;--==,,_))))--___--~~~--__--~~--__----~~~'`=__-~+_-_. +# (@) (@) ````` `-_(())_-~ +# +# ,---. .=-.-..-._ ,-,--. +# _..---. .-.,.---. .--.' \ /==/_ /==/ \ .-._ ,-.'- _\ +# .' .'.-. \ /==/ ` \ \==\-/\ \ |==|, ||==|, \/ /, /==/_ ,_.' +# /==/- '=' /|==|-, .=., |/==/-|_\ | |==| ||==|- \| |\==\ \ +# |==|-, ' |==| '=' /\==\, - \ |==|- ||==| , | -| \==\ -\ +# |==| .=. \|==|- , .' /==/ - ,| |==| ,||==| - _ | _\==\ ,\ +# /==/- '=' ,|==|_ . ,'./==/- /\ - \|==|- ||==| /\ , |/==/\/ _ | +# |==| - //==/ /\ , )==\ _.\=\.-'/==/. //==/, | |- |\==\ - , / +# `-._`.___,' `--`-`--`--' `--` `--`-` `--`./ `--` `--`---' + +User-agent: * +Disallow: /media_proxy/ +Disallow: /interact/ +