Update rules to explicitly Allow: /

Eventual-Inc · Dec 20, 2023 · 39e8e8e · 39e8e8e
1 parent f26a0ac
commit 39e8e8e
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/docs/source/_html/robots.txt b/docs/source/_html/robots.txt
@@ -1,9 +1,16 @@
 User-agent: *
 
-Disallow: /projects/docs
+# Allow crawling of all paths
+Allow: /
 
-Allow: /projects/docs/en/latest
+# Override to disallow crawling of documentation
+Disallow: /projects/docs/
 
-Allow: /projects/docs/en/stable
+# Override again to allow crawling of /latest documentation
+Allow: /projects/docs/en/latest/
 
+# Override again to allow crawling of /stable documentation
+Allow: /projects/docs/en/stable/
+
+# Custom sitemap
 Sitemap: https://www.getdaft.io/sitemap-override.xml
diff --git a/docs/source/_html/sitemap-override.xml b/docs/source/_html/sitemap-override.xml
@@ -11,4 +11,9 @@
         <changefreq>weekly</changefreq>
         <priority>0.9</priority>
     </url>
+    <url>
+        <loc>https://www.getdaft.io/</loc>
+        <changefreq>monthly</changefreq>
+        <priority>0.5</priority>
+    </url>
 </urlset>