Merge pull request #41 from NeuromatchAcademy/robots.txt-update-action

robots.txt updating action

.github/workflows/update-robots-txt.yml

@@ -0,0 +1,34 @@
+name: Update robots.txt
+
+on: 
+  schedule:
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+
+jobs:
+  update-robots:
+    runs-on: ubuntu-latest
+    steps: 
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Update robot.txt
+        id: update
+        env:
+          API_KEY: ${{ secrets.ROBOTS }}
+        run: |
+          cp robots-base.txt public/robots.txt
+          curl --location 'https://api.darkvisitors.com/robots-txts' \
+          --header 'Content-Type: application/json' \
+          --header "Authorization: Bearer $API_KEY" \
+          --data '{ "agent_types": [ "AI Data Scraper", "AI Assistant", "AI Search Crawler" ], "disallow": "/" }' >> public/robots.txt
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: robots.txt-update
+          title: "Update robots.txt"
+          commit-message: "Update robots.txt"
+          labels: 'robots.txt'
+          add-paths: public/robots.txt
+          reviewers: hobgoblina,mannazsci,sneakers-the-rat
+          body: This PR was generated by the `Update robots.txt` action and contains updates to our robots.txt file, pulled from [Dark Visitors](https://darkvisitors.com/).

robots-base.txt

@@ -0,0 +1,22 @@
+#         .__---~~~(~~-_.
+#      _-'  ) -~~- ) _-" )_
+#     (  ( `-,_..`.,_--_ '_,)_
+#    (  -_)  ( -_-~  -_ `,    )
+#    (_ -_ _-~-__-~`, ,' )__-'))--___--~~~--__--~~--___--__..
+#    _ ~`_-'( (____;--==,,_))))--___--~~~--__--~~--__----~~~'`=__-~+_-_.
+#   (@) (@) `````      `-_(())_-~
+#
+#                            ,---.       .=-.-..-._          ,-,--.  
+#       _..---.   .-.,.---.  .--.'  \     /==/_ /==/ \  .-._ ,-.'-  _\ 
+#     .' .'.-. \ /==/  `   \ \==\-/\ \   |==|, ||==|, \/ /, /==/_ ,_.' 
+#    /==/- '=' /|==|-, .=., |/==/-|_\ |  |==|  ||==|-  \|  |\==\  \    
+#    |==|-,   ' |==|   '='  /\==\,   - \ |==|- ||==| ,  | -| \==\ -\   
+#    |==|  .=. \|==|- ,   .' /==/ -   ,| |==| ,||==| -   _ | _\==\ ,\  
+#    /==/- '=' ,|==|_  . ,'./==/-  /\ - \|==|- ||==|  /\ , |/==/\/ _ | 
+#   |==|   -   //==/  /\ ,  )==\ _.\=\.-'/==/. //==/, | |- |\==\ - , / 
+#   `-._`.___,' `--`-`--`--' `--`        `--`-` `--`./  `--` `--`---'
+
+User-agent: *
+Disallow: /media_proxy/
+Disallow: /interact/
+