Merge pull request #567 from CityOfNewYork/johnyu95-fix-update-reques…

…t-statuses-job

Fixed Update Request Statuses Job

app/jobs.py

@@ -7,12 +7,12 @@
 from sqlalchemy.exc import SQLAlchemyError
 
 from app import calendar, sentry, db
-from app.constants import OPENRECORDS_DL_EMAIL, request_status
+from app.constants import OPENRECORDS_DL_EMAIL, request_status, determination_type
 from app.constants.event_type import EMAIL_NOTIFICATION_SENT, REQ_STATUS_CHANGED
 from app.constants.response_privacy import PRIVATE
 from app.lib.db_utils import create_object, update_object
 from app.lib.email_utils import send_email
-from app.models import Agencies, Emails, Events, Requests
+from app.models import Agencies, Emails, Events, Requests, Responses, Determinations
 
 # NOTE: (For Future Reference)
 # If we find ourselves in need of a request context, app.test_request_context() might come in handy.
@@ -43,81 +43,104 @@ def _update_request_statuses():
     due_soon_date = calendar.addbusdays(
         now, current_app.config['DUE_SOON_DAYS_THRESHOLD']
     ).replace(hour=23, minute=59, second=59)  # the entire day
+    request_errors = []
 
     agencies = Agencies.query.with_entities(Agencies.ein).filter_by(is_active=True).all()
     for agency_ein, in agencies:
+        # Overdue requests
         requests_overdue = Requests.query.filter(
             Requests.due_date < now,
             Requests.status != request_status.CLOSED,
             Requests.agency_ein == agency_ein
         ).order_by(
-            Requests.due_date.asc()
+            Requests.id.asc()
         ).all()
 
+        # Query for all acknowledged overdue requests
+        agency_requests_overdue = Requests.query.join(Responses, Determinations).filter(
+            Requests.due_date < now,
+            Requests.status != request_status.CLOSED,
+            Requests.agency_ein == agency_ein,
+            Determinations.dtype == determination_type.ACKNOWLEDGMENT
+        ).order_by(
+            Requests.id.asc()
+        ).all()
+
+        # Get the difference for all unacknowledged overdue requests
+        agency_acknowledgments_overdue = list(set(requests_overdue) - set(agency_requests_overdue))
+        agency_acknowledgments_overdue.sort(key=lambda x: x.id)
+
+        # Due soon requests
         requests_due_soon = Requests.query.filter(
             Requests.due_date > now,
             Requests.due_date <= due_soon_date,
             Requests.status != request_status.CLOSED,
             Requests.agency_ein == agency_ein
         ).order_by(
-            Requests.due_date.asc()
+            Requests.id.asc()
+        ).all()
+
+        # Query for all acknowledged due soon requests
+        agency_requests_due_soon = Requests.query.join(Responses, Determinations).filter(
+            Requests.due_date > now,
+            Requests.due_date <= due_soon_date,
+            Requests.status != request_status.CLOSED,
+            Requests.agency_ein == agency_ein,
+            Determinations.dtype == determination_type.ACKNOWLEDGMENT
+        ).order_by(
+            Requests.id.asc()
         ).all()
 
+        # Get the difference for all unacknowledged due soon requests
+        agency_acknowledgments_due_soon = list(set(requests_due_soon) - set(agency_requests_due_soon))
+        agency_acknowledgments_due_soon.sort(key=lambda x: x.id)
+
         if not requests_overdue and not requests_due_soon:
             continue
 
-        agency_requests_overdue = []
-        agency_acknowledgments_overdue = []
-        agency_requests_due_soon = []
-        agency_acknowledgments_due_soon = []
-
         # OVERDUE
         for request in requests_overdue:
-
-            if request.was_acknowledged:
-                agency_requests_overdue.append(request)
-            else:
-                agency_acknowledgments_overdue.append(request)
-
             if request.status != request_status.OVERDUE:
-                create_object(
-                    Events(
-                        request.id,
-                        user_guid=None,
-                        type_=REQ_STATUS_CHANGED,
-                        previous_value={"status": request.status},
-                        new_value={"status": request_status.OVERDUE},
-                        response_id=None,
+                try:
+                    update_object(
+                        {"status": request_status.OVERDUE},
+                        Requests,
+                        request.id)
+                    create_object(
+                        Events(
+                            request.id,
+                            user_guid=None,
+                            type_=REQ_STATUS_CHANGED,
+                            previous_value={"status": request.status},
+                            new_value={"status": request_status.OVERDUE},
+                            response_id=None,
+                        )
                     )
-                )
-                update_object(
-                    {"status": request_status.OVERDUE},
-                    Requests,
-                    request.id)
+                except Exception:
+                    request_errors.append(
+                        (request.id, traceback.format_exc().replace("\n", "<br/>").replace(" ", "&nbsp;")))
 
         # DUE SOON
         for request in requests_due_soon:
-
-            if request.was_acknowledged:
-                agency_requests_due_soon.append(request)
-            else:
-                agency_acknowledgments_due_soon.append(request)
-
             if request.status != request_status.DUE_SOON:
-                create_object(
-                    Events(
-                        request.id,
-                        user_guid=None,
-                        type_=REQ_STATUS_CHANGED,
-                        previous_value={"status": request.status},
-                        new_value={"status": request_status.DUE_SOON},
-                        response_id=None,
+                try:
+                    update_object(
+                        {"status": request_status.DUE_SOON},
+                        Requests,
+                        request.id)
+                    create_object(
+                        Events(
+                            request.id,
+                            user_guid=None,
+                            type_=REQ_STATUS_CHANGED,
+                            previous_value={"status": request.status},
+                            new_value={"status": request_status.DUE_SOON},
+                            response_id=None,
+                        )
                     )
-                )
-                update_object(
-                    {"status": request_status.DUE_SOON},
-                    Requests,
-                    request.id)
+                except Exception:
+                    request_errors.append(
+                        (request.id, traceback.format_exc().replace("\n", "<br/>").replace(" ", "&nbsp;")))
 
         # mail to agency admins for each agency
         user_emails = list(set(admin.notification_email or admin.email for admin
@@ -159,6 +182,13 @@ def _update_request_statuses():
                 timestamp=datetime.utcnow()
             )
         )
+    send_email(
+        'Update Request Statuses Job Finished',
+        to=[OPENRECORDS_DL_EMAIL],
+        template='email_templates/email_update_request_statuses_job_finished',
+        timestamp=str(datetime.utcnow()),
+        request_errors=request_errors
+    )
 
 
 @celery.task(autoretry_for=(OperationalError, SQLAlchemyError,), retry_kwargs={'max_retries': 5}, retry_backoff=True)

app/main/views.py

@@ -57,12 +57,13 @@ def technical_support():
 
                 if recaptcha_response['success'] is False or recaptcha_response['score'] < current_app.config[
                     "RECAPTCHA_THRESHOLD"]:
+                    current_app.logger.exception("Recaptcha failed to verify response.\n\n{}".format(recaptcha_response))
                     flash('Recaptcha failed, please try again.', category='danger')
-                    render_template('main/contact.html')
+                    return render_template('main/contact.html')
             except:
                 current_app.logger.exception("Recaptcha failed to get a response.")
                 flash('Recaptcha failed, please try again.', category='danger')
-                render_template('main/contact.html')
+                return render_template('main/contact.html')
 
         name = request.form.get('name')
         email = request.form.get('email')

app/models.py

@@ -966,6 +966,14 @@ def agency_request_summary_released(self):
             and self.agency_request_summary_release_date < datetime.utcnow()
         )
 
+    @property
+    def requester_name(self):
+        try:
+            request_doc = es.get(index=current_app.config["ELASTICSEARCH_INDEX"], doc_type="request", id=self.id)
+            return request_doc['_source']['requester_name']
+        except Exception as e:
+            print(e)
+
     def es_update(self):
         if current_app.config["ELASTICSEARCH_ENABLED"]:
             if self.agency.is_active:

app/request/views.py

@@ -108,6 +108,7 @@ def new():
 
                 if recaptcha_response['success'] is False or recaptcha_response['score'] < current_app.config[
                     "RECAPTCHA_THRESHOLD"]:
+                    current_app.logger.exception("Recaptcha failed to verify response.\n\n{}".format(recaptcha_response))
                     flash('Recaptcha failed, please try again.', category='danger')
                     return render_template(
                         new_request_template,

app/templates/email_templates/email_request_status_changed.html

@@ -10,7 +10,7 @@
                     {% if request.custom_metadata %}
                         <li>Request Type: {{ request.description }}</li>
                     {% endif %}
-                    <li>Requester Last Name: {{ request.requester.last_name | title | safe }}</li>
+                    <li>Requester Name: {{ request.requester_name | title | safe }}</li>
                     <li>Request Opened: {{ request.date_submitted.strftime('%m/%d/%y') }}</li>
                 </ul>
             </li>
@@ -29,7 +29,7 @@
                         {% if request.custom_metadata %}
                             <li>Request Type: {{ request.description }}</li>
                         {% endif %}
-                        <li>Requester Last Name: {{ request.requester.last_name | title | safe }}</li>
+                        <li>Requester Name: {{ request.requester_name | title | safe }}</li>
                         <li>Request Opened: {{ request.date_submitted.strftime('%m/%d/%y') }}</li>
                     </ul>
                 </li>
@@ -55,7 +55,7 @@
                     {% if request.custom_metadata %}
                         <li>Request Type: {{ request.description }}</li>
                     {% endif %}
-                    <li>Requester Last Name: {{ request.requester.last_name | title | safe }}</li>
+                    <li>Requester Name: {{ request.requester_name | title | safe }}</li>
                     <li>Request Opened: {{ request.date_submitted.strftime('%m/%d/%y') }}</li>
                 </ul>
             </li>
@@ -81,7 +81,7 @@
                         {% if request.custom_metadata %}
                             <li>Request Type: {{ request.description }}</li>
                         {% endif %}
-                        <li>Requester Last Name: {{ request.requester.last_name | title | safe }}</li>
+                        <li>Requester Name: {{ request.requester_name | title | safe }}</li>
                         <li>Request Opened: {{ request.date_submitted.strftime('%m/%d/%y') }}</li>
                     </ul>
                 </li>

app/templates/email_templates/email_update_request_statuses_job_finished.html

@@ -0,0 +1,6 @@
+<p>{{ timestamp }}</p>
+
+{% for request in request_errors %}
+    <p><strong>{{ request[0] }}</strong></p>
+    <p>{{ request[1] | safe }}</p>
+{% endfor %}

openrecords.py

@@ -277,15 +277,7 @@ def extend_requests(agency_ein: str, agency_name: str, user_guid: str, extension
 
 @app.cli.command()
 def update_request_statuses():
-    try:
-        _update_request_statuses()
-    except Exception:
-        db.session.rollback()
-        send_email(
-            subject="Update Request Statuses Failure",
-            to=[OPENRECORDS_DL_EMAIL],
-            email_content=traceback.format_exc().replace("\n", "<br/>").replace(" ", "&nbsp;")
-        )
+    _update_request_statuses()
 
 
 @app.cli.command