added robots.txt

reader/views.py

@@ -131,6 +131,17 @@ def sitemap(request):
         return HttpResponse("Sitemap not found", status=404, content_type='text/plain')
 
 
+def robot(request):
+    # Define the path to the robots.txt file
+    filepath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'robots.txt')
+    try:
+        with open(filepath, 'r') as f:
+            robots_content = f.read()
+        return HttpResponse(robots_content, content_type='text/plain')
+    except FileNotFoundError:
+        return HttpResponse("robots.txt not found", status=404, content_type='text/plain')         
+
+
 def render_template(request, template_name='base.html', app_props=None, template_context=None, content_type=None, status=None, using=None):
     """
     This is a general purpose custom function that serves to render all the templates in the project and provide a central point for all similar processing.

robots.txt

@@ -0,0 +1,18 @@
+User-agent: *
+Disallow: /admin/
+Disallow: /accounts/
+Disallow: /api/
+Disallow: /settings/
+Disallow: /login/
+Disallow: /logout/
+Disallow: /register/
+Disallow: /reset/
+Disallow: /password_change/
+Disallow: /password_reset/
+Disallow: /search/
+
+Allow: /static/
+Allow: /media/
+
+Sitemap: https://pecha.org/sitemap.xml
+

sefaria/urls.py

@@ -25,6 +25,7 @@
 
 # App Pages
 urlpatterns = [
+    url(r'^robots.txt$', reader_views.robot, name='robots_txt'),
     url(r'^sitemap\.xml$', reader_views.sitemap, name='sitemap'),
     url(r'^$', reader_views.home, name="home"),
     url(r'^texts/?$', reader_views.texts_list, name="table_of_contents"),