upgrade step to add robots.txt rule for googlebot (#83)

upgrade step to add robots.txt rule for googlebot

CHANGES.rst

@@ -3,6 +3,9 @@ Changelog
 
 5.3.1 (unreleased)
 ------------------
+- Add upgrade step and setuphandler to fix robots.txt
+  original rules adding 'Allow: /*?expand*'
+  [lucabel]
 
 - Nothing changed yet.
 

src/redturtle/volto/profiles/default/metadata.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <metadata>
-  <version>4200</version>
+  <version>4300</version>
   <dependencies>
     <dependency>profile-plone.volto:default</dependency>
     <dependency>profile-plone.app.caching:with-caching-proxy</dependency>

src/redturtle/volto/setuphandlers.py

@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 from Products.CMFPlone.interfaces import INonInstallable
 from zope.interface import implementer
+from plone import api
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 @implementer(INonInstallable)
@@ -10,8 +14,56 @@ def getNonInstallableProfiles(self):
         return ["redturtle.volto:uninstall"]
 
 
+def upgrade_robots_txt(context):
+    robots = api.portal.get_registry_record("plone.robots_txt")
+    lines = robots.splitlines()
+
+    googlebot_user_agent = "User-Agent: Googlebot"
+    # I saw this writed also as googlebot, so:
+    normalized_google_bot_user_agent = googlebot_user_agent.lower().replace(" ", "")
+
+    useragent_string = "User-Agent: "
+    normalized_useragent_string = useragent_string.lower().replace(" ", "")
+
+    googlebot_allow_rule = "Allow: /*?expand*"
+
+    googlebot_index = -1
+    allow_rule_present = False
+    for i, line in enumerate(lines):
+        if line.lower().replace(" ", "") == normalized_google_bot_user_agent:
+            googlebot_index = i
+        elif (
+            googlebot_index != -1
+            and line.replace(" ", "").lower()  # noqa
+            == googlebot_allow_rule.replace(" ", "").lower()  # noqa
+        ):
+            allow_rule_present = True
+
+    if googlebot_index != -1 and not allow_rule_present:
+        # Trova l'indice della fine della sezione User-Agent: Googlebot
+        end_googlebot_index = googlebot_index + 1
+        while end_googlebot_index < len(lines) and not lines[
+            end_googlebot_index
+        ].lower().replace(" ", "").startswith(normalized_useragent_string):
+            end_googlebot_index += 1
+
+        # Aggiungi Allow: /*?expand* alla fine della sezione User-Agent: Googlebot
+        if lines[end_googlebot_index - 1] == "":
+            end_googlebot_index -= 1
+        lines.insert(end_googlebot_index, googlebot_allow_rule)
+
+        lines = "\n".join(lines)
+        api.portal.set_registry_record("plone.robots_txt", lines)
+        logger.info("Upgrade robots.txt with rule for googlebot")
+    else:
+        logger.info(
+            "Rule for Googlebot already present in robots.txt, no action needed"
+        )
+
+
 def post_install(context):
     """Post install script"""
+    upgrade_robots_txt(context)
 
 
 def uninstall(context):

src/redturtle/volto/upgrades.zcml

@@ -180,5 +180,13 @@
       handler=".upgrades.to_4200"
       />
 
+  <genericsetup:upgradeStep
+      title="Update robots"
+      description=""
+      profile="redturtle.volto:default"
+      source="4200"
+      destination="4300"
+      handler=".setuphandlers.upgrade_robots_txt"
+      />
 
 </configure>