Merge pull request #105 from dimagi/sk/backoff

handle server errors with backoff

commcare_export/cli.py

@@ -9,6 +9,7 @@
 import sys
 
 import dateutil.parser
+import requests
 from six.moves import input
 
 from commcare_export import excel_query
@@ -102,6 +103,7 @@ def main(argv):
                             format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
 
     logging.getLogger('alembic').setLevel(logging.WARN)
+    logging.getLogger('backoff').setLevel(logging.FATAL)
 
     if args.version:
         print('commcare-export version {}'.format(__version__))
@@ -264,11 +266,19 @@ def main_with_args(args):
     )
 
     with env:
-        lazy_result = query.eval(env)
-        if lazy_result is not None:
-            # evaluate lazy results
-            for r in lazy_result:
-                list(r) if r else r
+        try:
+            lazy_result = query.eval(env)
+            if lazy_result is not None:
+                # evaluate lazy results
+                for r in lazy_result:
+                    list(r) if r else r
+        except requests.exceptions.RequestException as e:
+            if e.response.status_code == 401:
+                print("\nAuthentication failed. Please check your credentials.")
+                return
+            else:
+                raise
+
 
     if checkpoint_manager:
         checkpoint_manager.set_final_checkpoint()

commcare_export/commcare_hq_client.py

@@ -2,12 +2,11 @@
 
 import logging
 from collections import OrderedDict
-from copy import deepcopy
 
+import backoff
 import requests
-from datetime import datetime
-from requests.auth import HTTPDigestAuth
 from requests.auth import AuthBase
+from requests.auth import HTTPDigestAuth
 
 AUTH_MODE_DIGEST = 'digest'
 AUTH_MODE_APIKEY = 'apikey'
@@ -27,6 +26,25 @@
 LATEST_KNOWN_VERSION='0.5'
 
 
+def on_backoff(details):
+    _log_backoff(details, 'Waiting for retry.')
+
+
+def on_giveup(details):
+    _log_backoff(details, 'Giving up.')
+
+
+def _log_backoff(details, action_message):
+    details['__suffix'] = action_message
+    logger.warn("Request failed after {tries} attempts ({elapsed:.1f}s). {__suffix}".format(**details))
+
+
+def is_client_error(ex):
+    if hasattr(ex, 'response') and ex.response:
+        return 400 <= ex.response.status_code < 500
+    return False
+
+
 class CommCareHqClient(object):
     """
     A connection to CommCareHQ for a particular version, project, and user.
@@ -67,6 +85,11 @@ def session(self, session):
     def api_url(self):
         return '%s/a/%s/api/v%s' % (self.url, self.project, self.version)
 
+    @backoff.on_exception(
+        backoff.expo, requests.exceptions.RequestException,
+        max_time=300, giveup=is_client_error,
+        on_backoff=on_backoff, on_giveup=on_giveup
+    )
     def get(self, resource, params=None):
         """
         Gets the named resource.
@@ -77,12 +100,9 @@ def get(self, resource, params=None):
         """
         logger.debug("Fetching batch: %s", params)
         resource_url = '%s/%s/' % (self.api_url, resource)
-        response = self.session.get(resource_url, params=params, auth=self.__auth)
-
-        if response.status_code != 200:
-            raise Exception('GET %s failed (%s): %s' % (resource_url, response.status_code, response.text))
-        else:
-            return response.json()
+        response = self.session.get(resource_url, params=params, auth=self.__auth, timeout=60)
+        response.raise_for_status()
+        return response.json()
 
     def iterate(self, resource, paginator, params=None):
         """

setup.py

@@ -79,6 +79,7 @@ def run_tests(self):
         'sqlalchemy',
         'pytz',
         'sqlalchemy-migrate',
+        'backoff'
     ],
     tests_require = ['pytest', 'psycopg2', 'mock'],
     cmdclass = {'test': PyTest},

tests/test_commcare_hq_client.py

@@ -13,7 +13,7 @@
 
 
 class FakeSession(object):
-    def get(self, resource_url, params=None, auth=None):
+    def get(self, resource_url, params=None, auth=None, timeout=None):
         result = self._get_results(params)
         # Mutatey construction method required by requests.Response
         response = requests.Response()