From e77ad8355dc1c2dcb024fb77f961d7e7b290ed4c Mon Sep 17 00:00:00 2001
From: Jay Chia <17691182+jaychia@users.noreply.github.com>
Date: Wed, 20 Dec 2023 16:59:24 +0800
Subject: [PATCH] Revert "[DOCS] Add proper robots.txt and sitemap.xml to index
 only latest and stable (#1752)"

This reverts commit 03f6dac818cb3288f17787594c55dedf0ce187b6.
---
 docs/source/_html/robots.txt           | 16 ----------------
 docs/source/_html/sitemap-override.xml | 19 -------------------
 docs/source/conf.py                    |  3 ---
 3 files changed, 38 deletions(-)
 delete mode 100644 docs/source/_html/robots.txt
 delete mode 100644 docs/source/_html/sitemap-override.xml

diff --git a/docs/source/_html/robots.txt b/docs/source/_html/robots.txt
deleted file mode 100644
index f5f081a8eb..0000000000
--- a/docs/source/_html/robots.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-User-agent: *
-
-# Allow crawling of all paths
-Allow: /
-
-# Override to disallow crawling of documentation
-Disallow: /projects/docs/
-
-# Override again to allow crawling of /latest documentation
-Allow: /projects/docs/en/latest/
-
-# Override again to allow crawling of /stable documentation
-Allow: /projects/docs/en/stable/
-
-# Custom sitemap
-Sitemap: https://www.getdaft.io/sitemap-override.xml
diff --git a/docs/source/_html/sitemap-override.xml b/docs/source/_html/sitemap-override.xml
deleted file mode 100644
index 0d27e95390..0000000000
--- a/docs/source/_html/sitemap-override.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
-        xmlns:xhtml="http://www.w3.org/1999/xhtml">
-    <url>
-        <loc>https://www.getdaft.io/projects/docs/en/latest/</loc>
-        <changefreq>daily</changefreq>
-        <priority>1</priority>
-    </url>
-    <url>
-        <loc>https://www.getdaft.io/projects/docs/en/stable/</loc>
-        <changefreq>weekly</changefreq>
-        <priority>0.9</priority>
-    </url>
-    <url>
-        <loc>https://www.getdaft.io/</loc>
-        <changefreq>monthly</changefreq>
-        <priority>0.5</priority>
-    </url>
-</urlset>
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 2a14d45aa0..b186fcb077 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -82,9 +82,6 @@
     "learn/10-min": "../10-min.html",
 }
 
-# Adds /_html folder which serves robots.txt and the sitemap
-html_extra_path = ["_html"]
-
 # Resolving code links to github
 # Adapted from: https://github.com/aaugustin/websockets/blob/778a1ca6936ac67e7a3fe1bbe585db2eafeaa515/docs/conf.py#L100-L134