From c60f96a65214d001ba929a84e9061c8009219658 Mon Sep 17 00:00:00 2001 From: Tony Oreglia Date: Wed, 27 Sep 2017 16:27:50 +1000 Subject: [PATCH] better action export error handling --- safetypy/safetypy.py | 20 +++++++++++--------- tools/exporter/exporter.py | 13 +++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/safetypy/safetypy.py b/safetypy/safetypy.py index 8312808..46b6f3a 100644 --- a/safetypy/safetypy.py +++ b/safetypy/safetypy.py @@ -388,26 +388,29 @@ def get_audit_actions(self, date_modified, offset=0, page_length=100): ) result = self.parse_json(response.content) if response.status_code == requests.codes.ok else None self.log_http_status(response.status_code, 'GET actions') - if result is None or None in [result.get('count'), result.get('offset'), result.get('total')]: + if result is None or None in [result.get('count'), result.get('offset'), result.get('total'), result.get('actions')]: return None return self.get_page_of_actions(logger, date_modified, result, offset, page_length) - def get_page_of_actions(self, logger, date_modified, result, offset=0, page_length=100): + def get_page_of_actions(self, logger, date_modified, previous_page, offset=0, page_length=100): """ Returns a page of action search results :param logger: the logger :param date_modified: fetch from that date onwards - :param result: a page of action search results + :param previous_page: a page of action search results :param offset: the index to start retrieving actions from :param page_length: the number of actions to return on the search page (max 100) :return: Array of action objects """ - if result['count'] + result['offset'] < result['total']: - logger.info('Paging Actions. Offset: ' + str(offset + page_length) + '. Total: ' + str(result['total'])) - return self.get_audit_actions(date_modified, offset + page_length) + result['actions'] - elif result['count'] + result['offset'] == result['total']: - return result['actions'] + if previous_page['count'] + previous_page['offset'] < previous_page['total']: + logger.info('Paging Actions. Offset: ' + str(offset + page_length) + '. Total: ' + str(previous_page['total'])) + next_page = self.get_audit_actions(date_modified, offset + page_length) + if next_page is None: + return None + return next_page + previous_page['actions'] + elif previous_page['count'] + previous_page['offset'] == previous_page['total']: + return previous_page['actions'] def get_audit(self, audit_id): """ @@ -431,7 +434,6 @@ def log_http_status(status_code, message): :param status_code: http status code to log :param message: to describe where the status code was obtained """ - logger = logging.getLogger('sp_logger') status_description = requests.status_codes._codes[status_code][0] log_string = str(status_code) + ' [' + status_description + '] status received ' + message diff --git a/tools/exporter/exporter.py b/tools/exporter/exporter.py index 6378d25..52569e3 100644 --- a/tools/exporter/exporter.py +++ b/tools/exporter/exporter.py @@ -719,7 +719,6 @@ def show_export_profiles_and_exit(list_export_profiles, sc_client): print(row_boundary) sys.exit(0) - def export_actions(logger, settings, sc_client): """ Export all actions created after date specified @@ -745,15 +744,13 @@ def sync_exports(logger, settings, sc_client): :param settings: Settings from command line and configuration file :param sc_client: Instance of SDK object """ - # GET list of audits - last_successful = get_last_successful(logger) - list_of_audits = sc_client.discover_audits(modified_after=last_successful) - # exporting actions if specified by user if 'actions' in settings[EXPORT_FORMATS]: export_actions(logger, settings, sc_client) - # if there are audits start exporting - if list_of_audits is not None and bool(set(settings[EXPORT_FORMATS]) & {'pdf', 'docx', 'csv', 'media', - 'web-report-link'}): + if not bool(set(settings[EXPORT_FORMATS]) & {'pdf', 'docx', 'csv', 'media', 'web-report-link'}): + return + last_successful = get_last_successful(logger) + list_of_audits = sc_client.discover_audits(modified_after=last_successful) + if list_of_audits is not None: logger.info(str(list_of_audits['total']) + ' audits discovered') export_count = 1 export_total = list_of_audits['total']