diff --git a/src/core/src/CoreMain.py b/src/core/src/CoreMain.py index 2380835f..86a624bb 100644 --- a/src/core/src/CoreMain.py +++ b/src/core/src/CoreMain.py @@ -101,6 +101,7 @@ def __init__(self, argv): patch_installer.mark_installation_completed() overall_patch_installation_operation_successful = True self.update_patch_substatus_if_pending(patch_operation_requested, overall_patch_installation_operation_successful, patch_assessment_successful, configure_patching_successful, status_handler, composite_logger) + status_handler.log_truncated_packages() except Exception as error: # Privileged operation handling for non-production use diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index 88a8be54..23c1b44a 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -176,6 +176,14 @@ class AutoAssessmentStates(EnumBackport): STATUS_SUCCESS = "Success" STATUS_WARNING = "Warning" + # Status file size + class StatusTruncationConfig(EnumBackport): + INTERNAL_FILE_SIZE_LIMIT_IN_BYTES = 126 * 1024 + AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES = 128 * 1024 + MIN_ASSESSMENT_PACKAGE_TO_RETAIN = 5 + TRUNCATION_WARNING_MESSAGE = "Package lists were truncated to limit reporting data volume. In-VM logs contain complete lists." + TURN_ON_TRUNCATION = True + # Wrapper-core handshake files EXT_STATE_FILE = 'ExtState.json' CORE_STATE_FILE = 'CoreState.json' @@ -268,6 +276,7 @@ class PatchAssessmentSummaryStartedBy(EnumBackport): class PatchOperationTopLevelErrorCode(EnumBackport): SUCCESS = 0 ERROR = 1 + WARNING = 2 class PatchOperationErrorCodes(EnumBackport): DEFAULT_ERROR = "ERROR" # default error code @@ -275,6 +284,7 @@ class PatchOperationErrorCodes(EnumBackport): PACKAGE_MANAGER_FAILURE = "PACKAGE_MANAGER_FAILURE" NEWER_OPERATION_SUPERSEDED = "NEWER_OPERATION_SUPERSEDED" UA_ESM_REQUIRED = "UA_ESM_REQUIRED" + TRUNCATION = "PACKAGE_LIST_TRUNCATED" ERROR_ADDED_TO_STATUS = "Error_added_to_status" diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index a6d70004..19e9803b 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -140,9 +140,7 @@ def __report_consolidated_configure_patch_status(self, status=Constants.STATUS_T self.status_handler.add_error_to_status(error_msg, Constants.PatchOperationErrorCodes.DEFAULT_ERROR, current_operation_override_for_error=Constants.CONFIGURE_PATCHING_AUTO_ASSESSMENT) # write consolidated status - self.status_handler.set_configure_patching_substatus_json(status=status, - automatic_os_patch_state=self.current_auto_os_patch_state, - auto_assessment_state=self.current_auto_assessment_state) + self.status_handler.set_configure_patching_substatus_json(status=status, automatic_os_patch_state=self.current_auto_os_patch_state, auto_assessment_state=self.current_auto_assessment_state) def __raise_if_telemetry_unsupported(self): if self.lifecycle_manager.get_vm_cloud_type() == Constants.VMCloudType.ARC and self.execution_config.operation not in [Constants.ASSESSMENT, Constants.INSTALLATION]: diff --git a/src/core/src/service_interfaces/StatusHandler.py b/src/core/src/service_interfaces/StatusHandler.py index 76c5c5ef..e4e5f72b 100644 --- a/src/core/src/service_interfaces/StatusHandler.py +++ b/src/core/src/service_interfaces/StatusHandler.py @@ -13,7 +13,9 @@ # limitations under the License. # # Requires Python 2.7+ +import datetime import collections +import copy import glob import json import os @@ -49,6 +51,9 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.__maintenance_window_exceeded = False self.__installation_reboot_status = Constants.RebootStatus.NOT_NEEDED self.__installation_packages_map = collections.OrderedDict() + self.__installation_substatus_msg_copy = None + self.__installation_packages_copy = [] + self.__installation_packages_removed = [] # Internal in-memory representation of Patch Assessment data self.__assessment_substatus_json = None @@ -57,6 +62,9 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.__assessment_errors = [] self.__assessment_total_error_count = 0 # All errors during assess, includes errors not in error objects due to size limit self.__assessment_packages_map = collections.OrderedDict() + self.__assessment_substatus_msg_copy = None + self.__assessment_packages_copy = [] + self.__assessment_packages_removed = [] # Internal in-memory representation of Patch Metadata for HealthStore self.__metadata_for_healthstore_substatus_json = None @@ -72,6 +80,9 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.__configure_patching_auto_assessment_errors = [] self.__configure_patching_auto_assessment_error_count = 0 # All errors relating to auto-assessment configuration. + # Internal in-memory representation of Truncated Patching data + self.__internal_file_capacity = Constants.StatusTruncationConfig.INTERNAL_FILE_SIZE_LIMIT_IN_BYTES + # Load the currently persisted status file into memory self.load_status_file_components(initial_load=True) @@ -106,6 +117,9 @@ def reset_assessment_data(self): self.__assessment_errors = [] self.__assessment_total_error_count = 0 self.__assessment_packages_map = collections.OrderedDict() + self.__assessment_packages_removed = [] + self.__assessment_packages_copy = [] + self.__assessment_substatus_msg_copy = None def set_package_assessment_status(self, package_names, package_versions, classification="Other", status="Available"): """ Externally available method to set assessment status for one or more packages of the **SAME classification and status** """ @@ -139,7 +153,7 @@ def set_package_assessment_status(self, package_names, package_versions, classif def sort_packages_by_classification_and_state(self, packages_list): """ Sorts a list of packages (usually either self.__assessment_packages or self.__installation_packages) by classification and patchState properties. (sorting order from highest priority to lowest): - 1. Classification: Security, Critical, Other, Unclassified + 1. Classification: Critical, Security, Other, Unclassified 2. Patch Installation State: Failed, Installed, Available, Pending, Excluded, NotSelected """ def sort_patch_state_key(x): @@ -344,24 +358,33 @@ def __new_assessment_summary_json(self, assessment_packages_json, status, code): # discern started by - either pure auto-assessment or assessment data being included with configure patching with assessmentMode set to AutomaticByPlatform started_by = Constants.PatchAssessmentSummaryStartedBy.PLATFORM if (self.execution_config.exec_auto_assess_only or self.execution_config.include_assessment_with_configure_patching) else Constants.PatchAssessmentSummaryStartedBy.USER - # Compose sub-status message - substatus_message = { - "assessmentActivityId": str(self.execution_config.activity_id), - "rebootPending": self.is_reboot_pending, - "criticalAndSecurityPatchCount": critsec_patch_count, - "otherPatchCount": other_patch_count, - "patches": assessment_packages_json, - "startTime": str(self.execution_config.start_time), - "lastModifiedTime": str(self.env_layer.datetime.timestamp()), - "startedBy": str(started_by), - "errors": self.__set_errors_json(self.__assessment_total_error_count, self.__assessment_errors) - } + # Compose substatus message + errors = self.__set_errors_json(self.__assessment_total_error_count, self.__assessment_errors) + + substatus_message = self.__compose_assessment_substatus_msg( + activity_id=self.execution_config.activity_id, reboot_pending=self.is_reboot_pending, crit_patch_count=critsec_patch_count, + other_patch_count=other_patch_count, packages=assessment_packages_json, start_time=self.execution_config.start_time, + last_modified_time=self.env_layer.datetime.timestamp(), started_by=started_by, errors=errors) if self.vm_cloud_type == Constants.VMCloudType.ARC: substatus_message["patchAssessmentStatus"] = code substatus_message["patchAssessmentStatusString"] = status + return substatus_message + def __compose_assessment_substatus_msg(self, activity_id, reboot_pending, crit_patch_count, other_patch_count, packages, start_time, last_modified_time, started_by, errors): + return { + "assessmentActivityId": str(activity_id), + "rebootPending": reboot_pending, + "criticalAndSecurityPatchCount": crit_patch_count, + "otherPatchCount": other_patch_count, + "patches": packages, + "startTime": str(start_time), + "lastModifiedTime": str(last_modified_time), + "startedBy": str(started_by), + "errors": errors + } + def set_installation_substatus_json(self, status=Constants.STATUS_TRANSITIONING, code=0): """ Prepare the deployment substatus json including the message containing deployment summary """ self.composite_logger.log_debug("Setting installation substatus. [Substatus={0}]".format(str(status))) @@ -405,20 +428,31 @@ def __new_installation_summary_json(self, installation_packages_json): self.__refresh_installation_reboot_status() # Compose substatus message + maintenance_run_id = self.execution_config.maintenance_run_id if self.execution_config.maintenance_run_id is not None else '' + errors = self.__set_errors_json(self.__installation_total_error_count, self.__installation_errors) + substatus_message = self.__compose_installation_substatus_msg(activity_id=self.execution_config.activity_id, reboot_status=self.__installation_reboot_status, + maintenance_window=self.__maintenance_window_exceeded, not_selected=not_selected_patch_count, excluded=excluded_patch_count, + pending=pending_patch_count, installed=installed_patch_count, failed=failed_patch_count, + packages=installation_packages_json, start_time=self.execution_config.start_time, + last_modified_time=self.env_layer.datetime.timestamp(), maintenance_id=maintenance_run_id, errors=errors) + + return substatus_message + + def __compose_installation_substatus_msg(self, activity_id, reboot_status, maintenance_window, not_selected, excluded, pending, installed, failed, packages, start_time, last_modified_time, maintenance_id, errors): return { - "installationActivityId": str(self.execution_config.activity_id), - "rebootStatus": str(self.__installation_reboot_status), - "maintenanceWindowExceeded": self.__maintenance_window_exceeded, - "notSelectedPatchCount": not_selected_patch_count, - "excludedPatchCount": excluded_patch_count, - "pendingPatchCount": pending_patch_count, - "installedPatchCount": installed_patch_count, - "failedPatchCount": failed_patch_count, - "patches": installation_packages_json, - "startTime": str(self.execution_config.start_time), - "lastModifiedTime": str(self.env_layer.datetime.timestamp()), - "maintenanceRunId": str(self.execution_config.maintenance_run_id) if self.execution_config.maintenance_run_id is not None else '', - "errors": self.__set_errors_json(self.__installation_total_error_count, self.__installation_errors) + "installationActivityId": str(activity_id), + "rebootStatus": str(reboot_status), + "maintenanceWindowExceeded": maintenance_window, + "notSelectedPatchCount": not_selected, + "excludedPatchCount": excluded, + "pendingPatchCount": pending, + "installedPatchCount": installed, + "failedPatchCount": failed, + "patches": packages, + "startTime": str(start_time), + "lastModifiedTime": str(last_modified_time), + "maintenanceRunId": str(maintenance_id), + "errors": errors } def set_patch_metadata_for_healthstore_substatus_json(self, status=Constants.STATUS_SUCCESS, code=0, patch_version=Constants.PATCH_VERSION_UNKNOWN, report_to_healthstore=False, wait_after_update=False): @@ -553,12 +587,18 @@ def load_status_file_components(self, initial_load=False): self.__installation_packages = [] self.__installation_errors = [] self.__installation_packages_map = collections.OrderedDict() + self.__installation_substatus_msg_copy = None + self.__installation_packages_copy = [] + self.__installation_packages_removed = [] self.__assessment_substatus_json = None self.__assessment_summary_json = None self.__assessment_packages = [] self.__assessment_errors = [] self.__assessment_packages_map = collections.OrderedDict() + self.__assessment_substatus_msg_copy = None + self.__assessment_packages_copy = [] + self.__assessment_packages_removed = [] self.__metadata_for_healthstore_substatus_json = None self.__metadata_for_healthstore_summary_json = None @@ -570,7 +610,7 @@ def load_status_file_components(self, initial_load=False): self.composite_logger.log_debug("Loading status file components [InitialLoad={0}].".format(str(initial_load))) - # Remove older complete status files + # Retain 10 complete status files, and remove older files self.__removed_older_complete_status_files(self.execution_config.status_folder) # Verify the status file exists - if not, reset status file @@ -596,6 +636,7 @@ def load_status_file_components(self, initial_load=False): self.__installation_substatus_json = complete_status_file_data['status']['substatus'][i] else: self.__installation_summary_json = self.__get_substatus_message(complete_status_file_data, i) + # Reload patches into installation ordered map for fast look up self.__installation_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__installation_summary_json['patches']) self.__installation_packages = list(self.__installation_packages_map.values()) self.__maintenance_window_exceeded = bool(self.__installation_summary_json['maintenanceWindowExceeded']) @@ -606,6 +647,7 @@ def load_status_file_components(self, initial_load=False): self.__installation_total_error_count = self.__get_total_error_count_from_prev_status(errors['message']) if name == Constants.PATCH_ASSESSMENT_SUMMARY: # if it exists, it must be to spec, or an exception will get thrown self.__assessment_summary_json = self.__get_substatus_message(complete_status_file_data, i) + # Reload patches into assessment ordered map for fast look up self.__assessment_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__assessment_summary_json['patches']) self.__assessment_packages = list(self.__assessment_packages_map.values()) errors = self.__assessment_summary_json['errors'] @@ -627,15 +669,16 @@ def load_status_file_components(self, initial_load=False): self.__configure_patching_errors = errors['details'] self.__configure_patching_top_level_error_count = self.__get_total_error_count_from_prev_status(errors['message']) - def __get_substatus_message(self, status_file_data, index): - return json.loads(status_file_data['status']['substatus'][index]['formattedMessage']['message']) + def __get_substatus_message(self, status_file_data, substatus_index): + """ Get the substatus payload message by index """ + return json.loads(status_file_data['status']['substatus'][substatus_index]['formattedMessage']['message']) def __load_complete_status_file_data(self, file_path): # Read the status file - raise exception on persistent failure for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): try: with self.env_layer.file_system.open(file_path, 'r') as file_handle: - complete_status_file_data = json.load(file_handle)[0] # structure is array of 1 + complete_status_file_data = json.load(file_handle)[0] # structure is array of 1 except Exception as error: if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: time.sleep(i + 1) @@ -697,10 +740,14 @@ def __write_status_file(self): shutil.rmtree(self.complete_status_file_path) # Write complete status file .complete.status - self.env_layer.file_system.write_with_retry_using_temp_file(self.complete_status_file_path, '[{0}]'.format(json.dumps(complete_status_payload)), mode='w+') + status_file_payload_json_dumps = json.dumps(complete_status_payload) + self.env_layer.file_system.write_with_retry_using_temp_file(self.complete_status_file_path, '[{0}]'.format(status_file_payload_json_dumps), mode='w+') + + if Constants.StatusTruncationConfig.TURN_ON_TRUNCATION: + status_file_payload_json_dumps = self.__check_file_size_and_timestamp_for_truncation(status_file_payload_json_dumps) - # Write agent status file - self.env_layer.file_system.write_with_retry_using_temp_file(self.status_file_path, '[{0}]'.format(json.dumps(complete_status_payload)), mode='w+') + # Write status file .status + self.env_layer.file_system.write_with_retry_using_temp_file(self.status_file_path, '[{0}]'.format(status_file_payload_json_dumps), mode='w+') # endregion # region - Error objects @@ -724,12 +771,8 @@ def add_error_to_status(self, message, error_code=Constants.PatchOperationErrorC if not message or Constants.ERROR_ADDED_TO_STATUS in message: return - formatted_message = self.__ensure_error_message_restriction_compliance(message) # Compose error detail - error_detail = { - "code": str(error_code), - "message": str(formatted_message) - } + error_detail = self.__set_error_detail(error_code, message) # determine if a current operation override has been requested current_operation = self.__current_operation if current_operation_override_for_error == Constants.DEFAULT_UNSPECIFIED_VALUE else current_operation_override_for_error @@ -798,17 +841,220 @@ def __try_add_error(error_list, detail): error_list.insert(0, detail) return True - def __set_errors_json(self, error_count_by_operation, errors_by_operation): + def __set_errors_json(self, error_count_by_operation, errors_by_operation, truncated=False): """ Compose the error object json to be added in 'errors' in given operation's summary """ + code = Constants.PatchOperationTopLevelErrorCode.SUCCESS if error_count_by_operation == 0 else Constants.PatchOperationTopLevelErrorCode.ERROR + + # Update the errors json to include truncation detail + if truncated: + error_count_by_operation += 1 # add 1 because of truncation + code = Constants.PatchOperationTopLevelErrorCode.WARNING if code != Constants.PatchOperationTopLevelErrorCode.ERROR else Constants.PatchOperationTopLevelErrorCode.ERROR + message = "{0} error/s reported.".format(error_count_by_operation) message += " The latest {0} error/s are shared in detail. To view all errors, review this log file on the machine: {1}".format(len(errors_by_operation), self.__log_file_path) if error_count_by_operation > 0 else "" return { - "code": Constants.PatchOperationTopLevelErrorCode.SUCCESS if error_count_by_operation == 0 else Constants.PatchOperationTopLevelErrorCode.ERROR, + "code": code, "details": errors_by_operation, "message": message } + + def __set_error_detail(self, error_code, message): + formatted_message = self.__ensure_error_message_restriction_compliance(message) + return { + "code": str(error_code), + "message": str(formatted_message) + } # endregion + # region - Patch Truncation + def log_truncated_packages(self): + """ log the removed packages from patches in CoreMain after main operation are marked completed """ + if not len(self.__assessment_packages_removed) == 0: + self.composite_logger.log_debug("Packages removed from assessment packages list: {0}".format(self.__assessment_packages_removed)) + if not len(self.__installation_packages_removed) == 0: + self.composite_logger.log_debug("Packages removed from installation packages list: {0}".format(self.__installation_packages_removed)) + if len(self.__assessment_packages_removed) == 0 and len(self.__installation_packages_removed) == 0: + self.composite_logger.log_debug("No packages truncated") + + def __check_file_size_and_timestamp_for_truncation(self, status_file_payload_json_dumps): + status_file_size_in_bytes = self.__calc_status_size_on_disk(status_file_payload_json_dumps) # calc complete_status_file_payload byte size on disk + + if status_file_size_in_bytes > self.__internal_file_capacity: # perform truncation complete_status_file byte size > 126kb + truncated_status_file = self.__create_truncated_status_file(status_file_size_in_bytes, status_file_payload_json_dumps) + status_file_payload_json_dumps = json.dumps(truncated_status_file) + + return status_file_payload_json_dumps + + def __create_truncated_status_file(self, status_file_size_in_bytes, complete_status_file_payload): + """ Truncate substatus message patch list when complete status file size is more than 126kb """ + """ + __create_truncated_status_file(self, status_file_size_in_bytes, complete_status_file_payload): + + truncated_status_file = json.loads(complete_status_file_payload) + low_pri_index = None + _index = self.__get_substatus_index() + status_file_without_package_list_size = __calc_package_payload_size_on_disk(size_of_constant_status_data(complete_status_file_payload)) + size_of_max_packages_allowed_in_status = 126kb - status_file_without_package_list_size + + if assessment_index is not none: + _substatus_msg_copy = __get_substatus_message(truncated_status_file, _index) + _packages_copy = _substatus_msg_copy['patches'] + + if installation_index is not none: + _substatus_msg_copy = __get_substatus_message(truncated_status_file, _index) + _packages_copy = _substatus_msg_copy['patches'] + low_pri_index = __get_installation_low_pri_index() + + while status_file_size_in_bytes > 126kb: + __apply_truncation_process() + __split_assessment_list() + __apply_truncation() + + __recompose_truncated_status_file() + __get_current_complete_status_errors() + __recompose_truncated_substatus_msg() + __recompose_substatus_msg_errors() + __create_assessment_tombstone_list() + __create_assessment_tombstone() + __recreate_assessment_summary_json() + + __recompose_truncated_status_file() + __get_current_complete_status_errors() + __recompose_truncated_substatus_msg() + __recompose_substatus_msg_errors() + __create_installation_tombstone + __recreate_installation_summary_json() + + status_file_size_in_bytes, status_file_agent_size_diff = __get_new_size_in_bytes_after_truncation(truncated_status_file) + size_of_max_packages_allowed_in_status -= status_file_agent_size_diff + """ + self.composite_logger.log_debug("Begin package list truncation") + truncated_status_file = json.loads(complete_status_file_payload) # reload payload into python object + low_pri_index = None + assessment_substatus_index = self.__get_substatus_index(Constants.PATCH_ASSESSMENT_SUMMARY, truncated_status_file['status']['substatus']) + installation_substatus_index = self.__get_substatus_index(Constants.PATCH_INSTALLATION_SUMMARY, truncated_status_file['status']['substatus']) + + if assessment_substatus_index is not None: # If assessment data exists + self.__assessment_substatus_msg_copy = self.__get_substatus_message(truncated_status_file, assessment_substatus_index) + self.__assessment_packages_copy = self.__assessment_substatus_msg_copy['patches'] + + if installation_substatus_index is not None: # If installation data exists + self.__installation_substatus_msg_copy = self.__get_substatus_message(truncated_status_file, installation_substatus_index) + self.__installation_packages_copy = self.__installation_substatus_msg_copy['patches'] + low_pri_index = self.__get_installation_low_pri_index(self.__installation_packages_copy) + + status_file_without_package_list_size = self.size_of_constant_status_data(copy.deepcopy(truncated_status_file), assessment_substatus_index, installation_substatus_index) # Deepcopy fully copy the object avoid reference modification + size_of_max_packages_allowed_in_status = self.__internal_file_capacity - status_file_without_package_list_size + + while status_file_size_in_bytes > self.__internal_file_capacity: + # Start truncation process + packages_retained_in_assessment, packages_removed_from_assessment, packages_retained_in_installation, packages_removed_from_installation = \ + self.__apply_truncation_process(self.__assessment_packages_copy, self.__installation_packages_copy, size_of_max_packages_allowed_in_status, low_pri_index) + + if len(packages_removed_from_assessment) > 0: + # Update current assessment # of removed packages + self.__assessment_packages_removed = packages_removed_from_assessment + assessment_tombstone_records = self.__create_assessment_tombstone_list(self.__assessment_packages_removed) + packages_retained_in_assessment.extend(assessment_tombstone_records) # Add assessment tombstone records + + # Recompose truncated status file payload (assessment) + truncated_status_file = self.__recompose_truncated_status_file(truncated_status_file=truncated_status_file, truncated_package_list=packages_retained_in_assessment, + count_total_errors=self.__assessment_total_error_count, truncated_substatus_msg=self.__assessment_substatus_msg_copy, substatus_index=assessment_substatus_index) + + if len(packages_removed_from_installation) > 0: + # Update current installation # of removed packages + self.__installation_packages_removed = packages_removed_from_installation + # Todo need further requirements to decompose installation tombstone by classifications + installation_tombstone_record = self.__create_installation_tombstone() + packages_retained_in_installation.append(installation_tombstone_record) # Add installation tombstone records + + # Recompose truncated status file payload (installation) + truncated_status_file = self.__recompose_truncated_status_file(truncated_status_file=truncated_status_file, truncated_package_list=packages_retained_in_installation, + count_total_errors=self.__installation_total_error_count, truncated_substatus_msg=self.__installation_substatus_msg_copy, substatus_index=installation_substatus_index) + + status_file_size_in_bytes = self.__calc_status_size_on_disk(json.dumps(truncated_status_file)) + status_file_agent_size_diff = status_file_size_in_bytes - self.__internal_file_capacity + size_of_max_packages_allowed_in_status -= status_file_agent_size_diff # Reduce the max packages byte size by tombstone, new error, and escape chars byte size + + self.composite_logger.log_debug("End package list truncation") + + return truncated_status_file + + def __split_assessment_list(self, assessment_packages): + """ Split package list, keep 5 minimum packages, and remaining packages for truncation """ + min_packages_count = Constants.StatusTruncationConfig.MIN_ASSESSMENT_PACKAGE_TO_RETAIN + min_assessment_patches_to_retain, assessment_patches_eligible_for_truncation = (assessment_packages[:min_packages_count], assessment_packages[min_packages_count:]) \ + if len(assessment_packages) > min_packages_count else (assessment_packages, []) + + return min_assessment_patches_to_retain, assessment_patches_eligible_for_truncation + + def __apply_truncation_process(self, assessment_packages, installation_packages, max_package_list_capacity, low_pri_index=None): + """ Truncation function call split assessment method and apply truncation on assessment and installation packages """ + installation_low_pri = [] + installation_high_pri = installation_packages + # Cut assessment list into [:5], [5:] + min_assessment_patches_to_retain, assessment_patches_eligible_for_truncation = self.__split_assessment_list(assessment_packages) + + if len(min_assessment_patches_to_retain) > 0: + max_package_list_capacity = max_package_list_capacity - self.__calc_package_payload_size_on_disk(min_assessment_patches_to_retain) + + # Apply high priority (Failed, Installed) and low priority (Pending, Excluded, Not_Selected) installation logic, and keep min 5 assessment packages + if low_pri_index: + installation_high_pri = installation_packages[:low_pri_index] + installation_low_pri = installation_packages[low_pri_index:] + + packages_retained_in_install_high_pri, packages_removed_from_inst_high_pri, remaining_list_capacity = self.__apply_truncation(installation_high_pri, max_package_list_capacity) + packages_retained_in_assessment, packages_removed_from_assessment, remaining_list_capacity = self.__apply_truncation(assessment_patches_eligible_for_truncation, remaining_list_capacity) + packages_retained_in_install_low_pri, packages_removed_from_inst_low_pri, _ = self.__apply_truncation(installation_low_pri, remaining_list_capacity) + + truncated_installation_list = packages_retained_in_install_high_pri + packages_retained_in_install_low_pri + packages_removed_from_installation = packages_removed_from_inst_high_pri + packages_removed_from_inst_low_pri + truncated_assessment_list = min_assessment_patches_to_retain + packages_retained_in_assessment + + return truncated_assessment_list, packages_removed_from_assessment, truncated_installation_list, packages_removed_from_installation + + def __get_installation_low_pri_index(self, priority_sorted_installation_packages): + """" Get the first index of Pending, Excluded, or Not_Selected installation packages """ + for index, package in enumerate(priority_sorted_installation_packages): + package_state = package['patchInstallationState'] + if Constants.PENDING in package_state or Constants.EXCLUDED in package_state or Constants.NOT_SELECTED in package_state: + return index + + return None + + def __apply_truncation(self, package_list, capacity): + """ Binary search + Instead of checking list[middel_index] >= target, check byte_size(list[:middle_index]), + as byte_size[list[:i]] is monotonically increasing, i.e. + byte_size[list[:1]] < byte_size[list[:2]] < byte_size[list[:3]] ... + return truncated_list, packages_removed_from_list, and remaining max_package_list_capacity + """ + left_index = 0 + right_index = len(package_list) - 1 + + # Empty list after 2xjson.dumps have 4-5 bytes, no truncation, keep list capacity as it is + if len(package_list) == 0: + return [], [], capacity + # check if package list byte size <= list capacity, then returns it (no truncation needed) + if self.__calc_package_payload_size_on_disk(package_list) <= capacity: + return package_list, [], capacity - self.__calc_package_payload_size_on_disk(package_list) + # Check if first element byte size in the list > remaining list capacity, then add package_list to packages_removed_from_list + if self.__calc_package_payload_size_on_disk(package_list[0]) > capacity: + return [], package_list, capacity + + while left_index < right_index: + mid_index = left_index + int((right_index - left_index) / 2) + if self.__calc_package_payload_size_on_disk(package_list[:mid_index]) >= capacity: + right_index = mid_index + else: + left_index = mid_index + 1 + + truncated_list = package_list[:left_index - 1] + packages_removed_from_list = package_list[left_index - 1:] + truncated_list_byte_size = self.__calc_package_payload_size_on_disk(truncated_list) + + return truncated_list, packages_removed_from_list, capacity - truncated_list_byte_size + def __removed_older_complete_status_files(self, status_folder): """ Retain 10 latest status complete file and remove other .complete.status files """ files_removed = [] @@ -827,3 +1073,111 @@ def __removed_older_complete_status_files(self, status_folder): self.composite_logger.log_debug("Cleaned up older complete status files: {0}".format(files_removed)) + def __calc_status_size_on_disk(self, status_file_dumps): + """ Calculate status file size in bytes on disk """ + return len(status_file_dumps.encode("utf-8")) + + def __calc_package_payload_size_on_disk(self, package_list): + """ Calculate final package list size in bytes (because of escape chars) """ + first_json_dump = json.dumps(package_list) + + return len(json.dumps(first_json_dump).encode("utf-8")) + + def size_of_constant_status_data(self, complete_status_file_payload, assessment_status_index, installation_status_index): + """ Get the size in bytes of the complete_status_file without packages data """ + status_file_no_list_data = complete_status_file_payload + if assessment_status_index is not None: + assessment_msg_without_packages = self.__update_substatus_msg(substatus_msg=self.__assessment_substatus_msg_copy, substatus_msg_patches=[]) + status_file_no_list_data['status']['substatus'][assessment_status_index]['formattedMessage']['message'] = json.dumps(assessment_msg_without_packages) + + if installation_status_index is not None: + installation_msg_without_packages = self.__update_substatus_msg(substatus_msg=self.__installation_substatus_msg_copy, substatus_msg_patches=[]) + status_file_no_list_data['status']['substatus'][installation_status_index]['formattedMessage']['message'] = json.dumps(installation_msg_without_packages) + + return self.__calc_status_size_on_disk(json.dumps(status_file_no_list_data)) + + def __get_substatus_index(self, substatus_list_name, substatus_list): + """" Get substatus index from the current substatus """ + for substatus_index, substatus_name in enumerate(substatus_list): + if substatus_name['name'] == substatus_list_name: + return substatus_index + + return None + + def __recompose_truncated_status_file(self, truncated_status_file, truncated_package_list, count_total_errors, truncated_substatus_msg, substatus_index): + """ Recompose final truncated status file version """ + truncated_detail_list = [] + code, errors_details = self.__get_current_complete_status_errors(substatus_msg=truncated_substatus_msg) + + # Check for existing errors before recompose + if code != Constants.PatchOperationTopLevelErrorCode.ERROR: + truncated_status_file['status']['substatus'][substatus_index]['status'] = Constants.STATUS_WARNING.lower() # Update substatus status to warning + else: + truncated_detail_list.extend(errors_details) + + truncated_msg_errors = self.__recompose_substatus_msg_errors(truncated_detail_list, count_total_errors) # Recompose substatus msg errors + truncated_substatus_msg = self.__update_substatus_msg(substatus_msg=truncated_substatus_msg, substatus_msg_patches=truncated_package_list, substatus_msg_errors=truncated_msg_errors) + truncated_status_file['status']['substatus'][substatus_index]['formattedMessage']['message'] = json.dumps(truncated_substatus_msg) + + return truncated_status_file + + def __recompose_substatus_msg_errors(self, truncation_detail_list, count_total_errors): + """ Recompose truncated substatus errors json """ + error_msg = Constants.StatusTruncationConfig.TRUNCATION_WARNING_MESSAGE + truncated_error_detail = self.__set_error_detail(Constants.PatchOperationErrorCodes.TRUNCATION, error_msg) # Reuse the errors object set up + self.__try_add_error(truncation_detail_list, truncated_error_detail) + truncated_errors_json = self.__set_errors_json(count_total_errors, truncation_detail_list, True) # True for truncated + + return truncated_errors_json + + def __update_substatus_msg(self, substatus_msg, substatus_msg_patches, substatus_msg_errors=None): + substatus_msg['patches'] = substatus_msg_patches + if substatus_msg_errors: + substatus_msg['errors'] = substatus_msg_errors + + return substatus_msg + + def __get_current_complete_status_errors(self, substatus_msg): + """ Get the complete status file errors code and errors details """ + return substatus_msg['errors']['code'], substatus_msg['errors']['details'] + + def __create_assessment_tombstone_list(self, packages_removed_from_assessment): + assessment_tombstone_map = {} + tombstone_record_list = [] + + # Map['classification', count] + for package in packages_removed_from_assessment: + classifications = package['classifications'][0] + assessment_tombstone_map[classifications] = assessment_tombstone_map.get(classifications, 0) + 1 + + # Add assessment tombstone record per classifications except unclassified + for tombstone_classification, tombstone_package_count in assessment_tombstone_map.items(): + if not tombstone_classification == Constants.PackageClassification.UNCLASSIFIED: + tombstone_record_list.append(self.__create_assessment_tombstone(tombstone_package_count, tombstone_classification)) + + return tombstone_record_list + + def __create_assessment_tombstone(self, tombstone_packages_count, tombstone_classification): + """ Tombstone record for truncated assessment + Patch Name: 20 additional updates of classification reported. + Classification: [Critical, Security, Other] + """ + tombstone_name = str(tombstone_packages_count) + ' additional updates of classification ' + tombstone_classification + ' reported', + return { + 'patchId': 'Truncated_patch_list_id', + 'name': tombstone_name, + 'version': '0.0.0', + 'classifications': [tombstone_classification] + } + + def __create_installation_tombstone(self): + """ Tombstone record for truncated installation """ + return { + 'patchId': 'Truncated_patch_list_id', + 'name': 'Truncated_patch_list', + 'version': '0.0.0', + 'classifications': ['Other'], + 'patchInstallationState': 'NotSelected' + } + # endregion + diff --git a/src/core/tests/Test_CoreMain.py b/src/core/tests/Test_CoreMain.py index 922fa900..e02a236c 100644 --- a/src/core/tests/Test_CoreMain.py +++ b/src/core/tests/Test_CoreMain.py @@ -17,11 +17,11 @@ import glob import json import os +import random import re import shutil import unittest import uuid - from core.src.CoreMain import CoreMain from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer @@ -74,6 +74,7 @@ def test_operation_fail_for_non_autopatching_request(self): self.assertEqual(len(json.loads(substatus_file_data[1]["formattedMessage"]["message"])["errors"]["details"]), 1) self.assertTrue(substatus_file_data[2]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_operation_fail_for_autopatching_request(self): @@ -102,6 +103,7 @@ def test_operation_fail_for_autopatching_request(self): self.assertFalse(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_operation_success_for_non_autopatching_request(self): @@ -123,6 +125,7 @@ def test_operation_success_for_non_autopatching_request(self): self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) self.assertTrue(substatus_file_data[2]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_operation_success_for_autopatching_request(self): @@ -152,6 +155,7 @@ def test_operation_success_for_autopatching_request(self): self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_operation_success_for_autopatching_request_with_security_classification(self): @@ -190,6 +194,7 @@ def test_operation_success_for_autopatching_request_with_security_classification self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_invalid_maintenance_run_id(self): @@ -219,6 +224,7 @@ def test_invalid_maintenance_run_id(self): self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() # test with a random string for maintenance run id @@ -247,6 +253,7 @@ def test_invalid_maintenance_run_id(self): self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_assessment_operation_success(self): @@ -267,6 +274,7 @@ def test_assessment_operation_success(self): self.assertTrue(substatus_file_data[0]["status"].lower() == Constants.STATUS_SUCCESS.lower()) self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_assessment_operation_fail(self): @@ -288,6 +296,7 @@ def test_assessment_operation_fail(self): self.assertEqual(len(json.loads(substatus_file_data[0]["formattedMessage"]["message"])["errors"]["details"]), 2) self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_assessment_operation_fail_due_to_no_telemetry(self): @@ -309,6 +318,7 @@ def test_assessment_operation_fail_due_to_no_telemetry(self): self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_ERROR.lower()) self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[1]["formattedMessage"]["message"])["errors"]["details"][0]["message"]) + runtime.stop() def test_installation_operation_fail_due_to_telemetry_unsupported_no_events_folder(self): @@ -338,6 +348,7 @@ def test_installation_operation_fail_due_to_telemetry_unsupported_no_events_fold self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_ERROR.lower()) self.assertEqual(len(json.loads(substatus_file_data[3]["formattedMessage"]["message"])["errors"]["details"]), 1) self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[3]["formattedMessage"]["message"])["errors"]["details"][0]["message"]) + runtime.stop() def test_installation_operation_fail_due_to_no_telemetry(self): @@ -368,6 +379,7 @@ def test_installation_operation_fail_due_to_no_telemetry(self): self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_ERROR.lower()) self.assertEqual(len(json.loads(substatus_file_data[3]["formattedMessage"]["message"])["errors"]["details"]), 1) self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[3]["formattedMessage"]["message"])["errors"]["details"][0]["message"]) + runtime.stop() def test_assessment_operation_fail_on_arc_due_to_no_telemetry(self): @@ -388,6 +400,7 @@ def test_assessment_operation_fail_on_arc_due_to_no_telemetry(self): self.assertTrue(substatus_file_data[1]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_ERROR.lower()) self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[1]["formattedMessage"]["message"])["errors"]["details"][0]["message"]) + runtime.stop() def test_installation_operation_fail_on_arc_due_to_no_telemetry(self): @@ -417,6 +430,7 @@ def test_installation_operation_fail_on_arc_due_to_no_telemetry(self): self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_ERROR.lower()) self.assertEqual(len(json.loads(substatus_file_data[3]["formattedMessage"]["message"])["errors"]["details"]), 1) self.assertTrue(Constants.TELEMETRY_NOT_COMPATIBLE_ERROR_MSG in json.loads(substatus_file_data[3]["formattedMessage"]["message"])["errors"]["details"][0]["message"]) + runtime.stop() def test_install_all_packages_for_centos_autopatching(self): @@ -464,6 +478,7 @@ def test_install_all_packages_for_centos_autopatching(self): self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = backup_envlayer_platform_linux_distribution @@ -514,6 +529,7 @@ def test_install_all_packages_for_centos_autopatching_as_warning_with_never_rebo self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = backup_envlayer_platform_linux_distribution @@ -569,6 +585,7 @@ def test_install_only_critical_and_security_packages_for_redhat_autopatching(sel self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = backup_envlayer_platform_linux_distribution @@ -625,6 +642,7 @@ def test_install_only_critical_and_security_packages_for_redhat_autopatching_war self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = backup_envlayer_platform_linux_distribution @@ -930,6 +948,7 @@ def test_assessment_operation_fail_after_package_manager_reboot(self): self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_TRANSITIONING.lower()) self.assertTrue(substatus_file_data[2]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_assessment_operation_success_after_package_manager_reboot(self): @@ -966,6 +985,7 @@ def test_assessment_operation_success_after_package_manager_reboot(self): self.assertTrue(substatus_file_data[1]["status"].lower() == Constants.STATUS_TRANSITIONING.lower()) self.assertTrue(substatus_file_data[2]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) + runtime.stop() def test_assessment_superseded(self): @@ -1041,6 +1061,7 @@ def test_temp_folder_created_during_execution_config_init(self): # validate temp_folder is created self.assertTrue(runtime.execution_config.temp_folder is not None) self.assertTrue(os.path.exists(runtime.execution_config.temp_folder)) + runtime.stop() # temp_folder is set to None in ExecutionConfig with a valid config_folder location @@ -1052,6 +1073,7 @@ def test_temp_folder_created_during_execution_config_init(self): # validate temp_folder is created self.assertTrue(runtime.execution_config.temp_folder is not None) self.assertTrue(os.path.exists(runtime.execution_config.temp_folder)) + runtime.stop() # temp_folder is set to None in ExecutionConfig with an invalid config_folder location, throws exception @@ -1066,6 +1088,7 @@ def test_temp_folder_created_during_execution_config_init(self): # validate temp_folder is not created self.assertFalse(os.path.exists(os.path.join(os.path.curdir, "scratch", "tmp"))) os.path.exists = backup_os_path_exists + runtime.stop() def test_delete_temp_folder_contents_success(self): @@ -1083,6 +1106,7 @@ def test_delete_temp_folder_contents_success(self): self.assertTrue(argument_composer.temp_folder is not None) files_matched = glob.glob(str(argument_composer.temp_folder) + "/" + str(Constants.TEMP_FOLDER_CLEANUP_ARTIFACT_LIST)) self.assertTrue(len(files_matched) == 0) + runtime.stop() def test_delete_temp_folder_contents_when_none_exists(self): @@ -1098,6 +1122,7 @@ def test_delete_temp_folder_contents_when_none_exists(self): self.assertTrue(runtime.execution_config.temp_folder is not None) files_matched = glob.glob(str(runtime.execution_config.temp_folder) + "/" + str(Constants.TEMP_FOLDER_CLEANUP_ARTIFACT_LIST)) self.assertTrue(len(files_matched) == 0) + runtime.stop() def test_delete_temp_folder_contents_failure(self): @@ -1108,7 +1133,6 @@ def test_delete_temp_folder_contents_failure(self): # mock os.remove() self.backup_os_remove = os.remove os.remove = self.mock_os_remove - argument_composer.operation = Constants.ASSESSMENT runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) @@ -1122,6 +1146,7 @@ def test_delete_temp_folder_contents_failure(self): # reset os.remove() mock os.remove = self.backup_os_remove + runtime.stop() def __check_telemetry_events(self, runtime): @@ -1134,6 +1159,711 @@ def __check_telemetry_events(self, runtime): self.assertTrue('Core' in events[0]['TaskName']) f.close() + def test_assessment_operation_truncation_under_size_limit(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.ASSESSMENT + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('HappyPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # HappyPath 3 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Security\"]}, + # {\"patchId\": \"libgcc_5.60.7-8.1_Ubuntu_16.04\", \"name\": \"libgcc\", \"version\": \"5.60.7-8.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + patch_count_for_test = random.randint(200, 432) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.ASSESSMENT) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test + 3) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertNotEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test + 3) + status_file_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertNotEqual(status_file_patches[1]['patchId'], "Truncated_patch_list_id") + self.assertTrue('additional updates of classification' not in status_file_patches[1]['name']) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], 0) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + self.assertFalse("review this log file on the machine" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + self.assertEqual(len(runtime.status_handler._StatusHandler__assessment_packages_removed), 0) + + runtime.stop() + + def test_assessment_operation_truncation_over_size_limit(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.ASSESSMENT + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('HappyPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # HappyPath add 3 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Security\"]}, + # {\"patchId\": \"libgcc_5.60.7-8.1_Ubuntu_16.04\", \"name\": \"libgcc\", \"version\": \"5.60.7-8.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions, "Critical") + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.ASSESSMENT) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test + 3) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-2]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-3]['patchId'], "Truncated_patch_list_id") # 3 tombstons Critical Security Other + self.assertTrue("additional updates of classification" in message_patches[-1]['name'][0]) + self.assertTrue(len(message_patches) < patch_count_for_test + 3) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + runtime.stop() + + def test_assessment_truncation_over_large_size_limit_for_extra_chars(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.ASSESSMENT + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('HappyPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # HappyPath add 3 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Security\"]}, + # {\"patchId\": \"libgcc_5.60.7-8.1_Ubuntu_16.04\", \"name\": \"libgcc\", \"version\": \"5.60.7-8.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + patch_count_for_test = 99997 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions, "Security") + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.ASSESSMENT) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test + 3) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertTrue(len(message_patches) < patch_count_for_test + 3) + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-1]['classifications'], ['Other']) + self.assertEqual(message_patches[-2]['classifications'], ['Security']) # 2 tombstones - Security, Other + self.assertTrue("additional updates of classification" in message_patches[-1]['name'][0]) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + runtime.stop() + + def test_installation_truncation_over_size_limit(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.INSTALLATION + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('SuccessInstallPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # SuccessInstallPath add 2 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + patch_count_for_assessment = random.randint(798, 1100) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_assessment) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + patch_count_for_installation = random.randint(500, 1100) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_installation) + runtime.status_handler.set_package_install_status(test_packages, test_package_versions) + runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + # Assessment summary + assessment_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_substatus["status"], Constants.STATUS_SUCCESS.lower()) + assessment_msg = json.loads(assessment_substatus["formattedMessage"]["message"]) + self.assertEqual(len(assessment_msg["patches"]), patch_count_for_assessment + 2) + self.assertEqual(len(assessment_msg["errors"]["details"]), 0) + + assessment_activity_id = assessment_msg['assessmentActivityId'] + assessment_reboot_pending = assessment_msg['rebootPending'] + assessment_crit_patch_count = assessment_msg['criticalAndSecurityPatchCount'] + assessment_other_patch_count = assessment_msg['otherPatchCount'] + assessment_start_time = assessment_msg['startTime'] + assessment_last_modified_time = assessment_msg['lastModifiedTime'] + assessment_started_by = assessment_msg['startedBy'] + + # Installation summary + installation_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_substatus["status"], Constants.STATUS_SUCCESS.lower()) + installation_msg = json.loads(installation_substatus["formattedMessage"]["message"]) + self.assertEqual(len(installation_msg["patches"]), patch_count_for_installation + 2) + self.assertEqual(len(installation_msg["errors"]["details"]), 0) + + installation_activity_id = installation_msg['installationActivityId'] + installation_reboot_status = installation_msg['rebootStatus'] + installation_maintenance_window = installation_msg['maintenanceWindowExceeded'] + installation_not_selected = installation_msg['notSelectedPatchCount'] + installation_excluded = installation_msg['excludedPatchCount'] + installation_pending = installation_msg['pendingPatchCount'] + installation_installed = installation_msg['installedPatchCount'] + installation_failed = installation_msg['failedPatchCount'] + installation_start_time = installation_msg['startTime'] + installation_last_modified_time = installation_msg['lastModifiedTime'] + installation_maintenance_id = installation_msg['maintenanceRunId'] + + # Test truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0] + + # Test assessment truncation + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + assessment_truncated_substatus = substatus_file_data["status"]["substatus"][0] + self.assertEqual(assessment_truncated_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + # tombstone record + truncated_assessment_msg = json.loads(assessment_truncated_substatus["formattedMessage"]["message"]) + self.assertTrue(len(truncated_assessment_msg["patches"]) < patch_count_for_assessment + 2) + self.assertEqual(truncated_assessment_msg["patches"][-1]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(truncated_assessment_msg["patches"][-1]['classifications'], ['Other']) # 1 tombstone - Other + self.assertNotEqual(truncated_assessment_msg["patches"][-2]['patchId'], 'Truncated_patch_list_id') + self.assertTrue('additional updates of classification' in truncated_assessment_msg["patches"][-1]['name'][0]) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["message"]) + + truncated_assessment_activity_id = truncated_assessment_msg['assessmentActivityId'] + truncated_assessment_reboot_pending = truncated_assessment_msg['rebootPending'] + truncated_assessment_crit_patch_count = truncated_assessment_msg['criticalAndSecurityPatchCount'] + truncated_assessment_other_patch_count = truncated_assessment_msg['otherPatchCount'] + truncated_assessment_start_time = truncated_assessment_msg['startTime'] + truncated_assessment_last_modified_time = truncated_assessment_msg['lastModifiedTime'] + truncated_assessment_started_by = truncated_assessment_msg['startedBy'] + + # validate all assessment other fields in the message object are equal in both status files + self.assertEqual(assessment_activity_id, truncated_assessment_activity_id) + self.assertEqual(assessment_reboot_pending, truncated_assessment_reboot_pending) + self.assertEqual(assessment_crit_patch_count, truncated_assessment_crit_patch_count) + self.assertEqual(assessment_other_patch_count, truncated_assessment_other_patch_count) + self.assertEqual(assessment_start_time, truncated_assessment_start_time) + self.assertEqual(assessment_last_modified_time, truncated_assessment_last_modified_time) + self.assertEqual(assessment_started_by, truncated_assessment_started_by) + + + # Test installation truncation + installation_truncated_substatus = substatus_file_data["status"]["substatus"][1] + self.assertEqual(installation_truncated_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + truncated_installation_msg = json.loads(installation_truncated_substatus["formattedMessage"]["message"]) + self.assertEqual(len(truncated_installation_msg["patches"]), 3) # 1 tombstone + self.assertEqual(truncated_installation_msg["patches"][-1]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(truncated_installation_msg["patches"][-1]['classifications'][0], 'Other') + self.assertNotEqual(truncated_installation_msg["patches"][-2]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["message"]) + + truncated_installation_activity_id = truncated_installation_msg['installationActivityId'] + truncated_installation_reboot_status = truncated_installation_msg['rebootStatus'] + truncated_installation_maintenance_window = truncated_installation_msg['maintenanceWindowExceeded'] + truncated_installation_not_selected = truncated_installation_msg['notSelectedPatchCount'] + truncated_installation_excluded = truncated_installation_msg['excludedPatchCount'] + truncated_installation_pending = truncated_installation_msg['pendingPatchCount'] + truncated_installation_installed = truncated_installation_msg['installedPatchCount'] + truncated_installation_failed = truncated_installation_msg['failedPatchCount'] + truncated_installation_start_time = truncated_installation_msg['startTime'] + truncated_installation_last_modified_time = truncated_installation_msg['lastModifiedTime'] + truncated_installation_maintenance_id = truncated_installation_msg['maintenanceRunId'] + + # validate all installation other fields in the message object are equal in both status files + self.assertEqual(installation_activity_id, truncated_installation_activity_id) + self.assertEqual(installation_reboot_status, truncated_installation_reboot_status) + self.assertEqual(installation_maintenance_window, truncated_installation_maintenance_window) + self.assertEqual(installation_not_selected, truncated_installation_not_selected) + self.assertEqual(installation_excluded, truncated_installation_excluded) + self.assertEqual(installation_pending, truncated_installation_pending) + self.assertEqual(installation_installed, truncated_installation_installed) + self.assertEqual(installation_failed, truncated_installation_failed) + self.assertEqual(installation_start_time, truncated_installation_start_time) + self.assertEqual(installation_last_modified_time, truncated_installation_last_modified_time) + self.assertEqual(installation_maintenance_id, truncated_installation_maintenance_id) + + runtime.stop() + + def test_installation_keep_min_5_assessment_size_limit(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.INSTALLATION + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('SuccessInstallPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # SuccessInstallPath add 2 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + patch_count_for_assessment = 3 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_assessment) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + patch_count_for_installation = 1000 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_installation) + runtime.status_handler.set_package_install_status(test_packages, test_package_versions) + runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + + # Assessment summary + assessment_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_assessment + 2) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Installation summary + installation_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_installation + 2) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + + # Test assessment truncation + assessment_truncated_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_truncated_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_truncated_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"]), 5) # no tombstone + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"][-1]['name'], 'python-samba2') + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], 0) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test installation truncation + installation_truncated_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_truncated_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + self.assertTrue(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"]) < patch_count_for_installation + 2) + + # tombstone record + message_patches = json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['name'], 'Truncated_patch_list') + self.assertEqual(message_patches[-1]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(message_patches[-1]['classifications'][0], 'Other') + self.assertNotEqual(message_patches[-2]['patchId'][0], 'Truncated_patch_list_id') + self.assertEqual(432, patch_count_for_installation + 3 - len(message_patches)) # 1 tombstone + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + + runtime.stop() + + def test_installation_truncation_with_only_install_packages_over_size_limit(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.INSTALLATION + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('SuccessInstallPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # SuccessInstallPath add 2 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + patch_count_for_assessment = 7 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_assessment) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + patch_count_for_installation = 1000 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_installation) + runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + + # Assessment summary + assessment_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_assessment + 2) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Installation summary + installation_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_installation + 2) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + + # Test assessment truncation + assessment_truncated_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_truncated_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertTrue(len(message_patches) < len(json.loads(assessment_substatus["formattedMessage"]["message"])["patches"])) # truncated message < before truncated message + self.assertEqual(message_patches[-1]['patchId'], 'Truncated_patch_list_id') # 1 tombstone + self.assertNotEqual(message_patches[-2]['patchId'], 'Truncated_patch_list_id') # 1 tombstone + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + + # Test installation truncation + installation_truncated_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_truncated_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + self.assertTrue(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"]) < patch_count_for_installation + 2) + message_patches = json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['name'], 'Truncated_patch_list') + self.assertEqual(message_patches[-1]['patchId'], 'Truncated_patch_list_id') # 1 tombstone + self.assertNotEqual(message_patches[-2]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(440, patch_count_for_installation + 3 - len(message_patches)) # 440 removed packages + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + + runtime.stop() + + def test_installation_truncation_over_size_limit_success_path(self): + argument_composer = ArgumentComposer() + argument_composer.operation = Constants.INSTALLATION + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('SuccessInstallPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # SuccessInstallPath add 2 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + patch_count_for_assessment = 19998 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_assessment) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + patch_count_for_installation = 9998 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_installation) + runtime.status_handler.set_package_install_status(test_packages, test_package_versions) + runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + + # Assessment summary + assessment_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_assessment + 2) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Installation summary + installation_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_installation + 2) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data).encode('utf-8')) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + # Test assessment truncation + assessment_truncated_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_truncated_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + + # Tombstone record + message_patches = json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertNotEqual(message_patches[-2]['patchId'], "Truncated_patch_list_id") + self.assertTrue('additional updates of classification' in message_patches[-1]['name'][0]) + self.assertTrue(len(message_patches) < patch_count_for_assessment) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["message"]) + + # Test installation truncation + installation_truncated_substatus = substatus_file_data[0]["status"]["substatus"][1] + message_patches = json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(installation_truncated_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + self.assertEqual(3, len(message_patches)) # 1 tombstone + self.assertEqual(message_patches[-1]['patchId'], 'Truncated_patch_list_id') + self.assertNotEqual(message_patches[-2]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue('1 error/s reported. The latest 1 error/s are shared in detail.' in json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["message"]) + + runtime.stop() + + def test_installation_truncate_both_over_size_limit_happy_path(self): + argument_composer = ArgumentComposer() + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('HappyPath') + CoreMain(argument_composer.get_composed_arguments()) + + patch_count_for_assessment = random.randint(950, 1200) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_assessment) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + patch_count_for_installation = random.randint(875, 1200) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_installation) + runtime.status_handler.set_package_install_status(test_packages, test_package_versions) + runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_ERROR) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # HappyPath add 3 additional packages, HappyPath contains failed, pending, installed packages for installation + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Security\"]}, + # {\"patchId\": \"libgcc_5.60.7-8.1_Ubuntu_16.04\", \"name\": \"libgcc\", \"version\": \"5.60.7-8.1\", \"classifications\": [\"Other\"]}, + # {\"patchId\": \"libgoa-1_0-0_3.20.5-9.6_Ubuntu_16.04\", \"name\": \"libgoa-1_0-0\", \"version\": \"3.20.5-9.6\", \"classifications\": [\"Other\"]} + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + + # Assessment summary + assessment_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_assessment + 3) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Installation summary + installation_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_substatus["status"], Constants.STATUS_ERROR.lower()) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_installation + 3) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + + # Test truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + # Test assessment truncation + assessment_truncated_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_truncated_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_truncated_substatus["status"], Constants.STATUS_WARNING.lower()) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"]), 699) # 1 tombstone + message_patches = json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[- 1]['patchId'], 'Truncated_patch_list_id') + self.assertTrue('additional updates of classification' in message_patches[-1]['name'][0]) + self.assertTrue(patch_count_for_assessment + 4 - len(message_patches) > 0) # more than 1 removed packages + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["message"]) + + # Test installation truncation + installation_truncated_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_truncated_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_truncated_substatus["status"], Constants.STATUS_ERROR.lower()) + message_patches = json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertTrue(len(message_patches) < patch_count_for_installation) # 1 tombstone + self.assertEqual(message_patches[-1]['patchId'], 'Truncated_patch_list_id') + self.assertNotEqual(message_patches[-2]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.ERROR) + self.assertEqual(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 2) + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + + runtime.stop() + + def test_installation_truncattion_with_error_over_size_limit(self): + """ assessment list > size limit but truncate installation < size limit with error""" + argument_composer = ArgumentComposer() + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('FailInstallPath') + CoreMain(argument_composer.get_composed_arguments()) + + # check telemetry events + self.__check_telemetry_events(runtime) + + # Test code add 2 additional packages + # {\"patchId\": \"kernel-default_4.4.49-92.11.1_Ubuntu_16.04\", \"name\": \"kernel-default\", \"version\": \"4.4.49-92.11.1\", \"classifications\": [\"Security\"]}, + + patch_count_for_assessment = 598 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_assessment) + runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + + runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + patch_count_for_installation = 318 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_installation) + runtime.status_handler.set_package_install_status(test_packages, test_package_versions) + + # Adding multiple exceptions + runtime.status_handler.add_error_to_status("exception0", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + runtime.status_handler.add_error_to_status("exception1", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + runtime.status_handler.add_error_to_status("exception2", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + runtime.status_handler.add_error_to_status("exception3", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + runtime.status_handler.add_error_to_status("exception4", Constants.PatchOperationErrorCodes.PACKAGE_MANAGER_FAILURE) + runtime.status_handler.add_error_to_status("exception5", Constants.PatchOperationErrorCodes.OPERATION_FAILED) + + runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_ERROR) + + # Test Complete status file + with runtime.env_layer.file_system.open(runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + + # Assessment summary + assessment_substatus = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(assessment_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_substatus["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_assessment + 2) + self.assertEqual(len(json.loads(assessment_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Installation summary + installation_substatus = substatus_file_data[0]["status"]["substatus"][1] + self.assertEqual(installation_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_substatus["status"], Constants.STATUS_ERROR.lower()) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["patches"]), patch_count_for_installation + 2) + self.assertEqual(len(json.loads(installation_substatus["formattedMessage"]["message"])["errors"]["details"]), 5) + + # Test truncated status file + with runtime.env_layer.file_system.open(runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.INTERNAL_FILE_SIZE_LIMIT_IN_BYTES) + # Test assessment truncation + assessment_truncated_substatus = substatus_file_data[0] + self.assertEqual(assessment_truncated_substatus["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(assessment_truncated_substatus["status"], Constants.STATUS_SUCCESS.lower()) + message_patches = json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(len(message_patches), patch_count_for_assessment + 2) + self.assertEqual(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.SUCCESS) + self.assertEqual(len(json.loads(assessment_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test installation truncation + installation_truncated_substatus = substatus_file_data[1] + installation_patches = json.loads(installation_truncated_substatus["formattedMessage"]["message"])["patches"] + self.assertEqual(installation_truncated_substatus["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(installation_truncated_substatus["status"], Constants.STATUS_ERROR.lower()) + self.assertTrue(len(installation_patches) < patch_count_for_installation + 2) + self.assertEqual(installation_patches[-1]['patchId'], 'Truncated_patch_list_id') # 1 tombstone + self.assertNotEqual(installation_patches[-2]['patchId'], 'Truncated_patch_list_id') + self.assertEqual(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["code"], 1) + self.assertEqual(len(json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["details"]), 5) + # 1 failed installed packages errors, and 1 truncation error, plus 6 exception error + self.assertTrue("8 error/s reported. The latest 5 error/s are shared in detail." in json.loads(installation_truncated_substatus["formattedMessage"]["message"])["errors"]["message"]) + + runtime.stop() + + def __set_up_packages_func(self, val): + test_packages = [] + test_package_versions = [] + + for i in range(0, val): + test_packages.append('python-samba' + str(i)) + test_package_versions.append('2:4.4.5+dfsg-2ubuntu5.4') + + return test_packages, test_package_versions if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/src/core/tests/Test_StatusHandler.py b/src/core/tests/Test_StatusHandler.py index 09e227db..99441c06 100644 --- a/src/core/tests/Test_StatusHandler.py +++ b/src/core/tests/Test_StatusHandler.py @@ -16,8 +16,12 @@ import datetime import glob import json -import os +import random +import time import unittest +import tempfile +import os +import sys from core.src.bootstrap.Constants import Constants from core.src.service_interfaces.StatusHandler import StatusHandler from core.tests.library.ArgumentComposer import ArgumentComposer @@ -128,6 +132,23 @@ def test_set_package_install_status_classification_not_set(self): self.assertTrue("python-samba_2:4.4.5+dfsg-2ubuntu5.4" in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][0]["patchId"])) self.assertTrue("Other" in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][2]["classifications"])) + def test_set_package_install_unknown_patch_state_recorded(self): + packages, package_versions = self.runtime.package_manager.get_all_updates() + self.runtime.status_handler.set_package_install_status(packages, package_versions, Constants.AVAILABLE) + + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), 3) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][0]["name"], "python-samba") + self.assertTrue("Other" in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][0]["classifications"])) + self.assertEqual("Available", str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][0]["patchInstallationState"])) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][1]["name"], "samba-common-bin") + self.assertTrue("Other" in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][1]["classifications"])) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][2]["name"], "samba-libs") + self.assertTrue("python-samba_2:4.4.5+dfsg-2ubuntu5.4" in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][0]["patchId"])) + self.assertTrue("Other" in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][2]["classifications"])) + def test_set_installation_reboot_status(self): self.assertRaises(Exception, self.runtime.status_handler.set_installation_reboot_status, "INVALID_STATUS") @@ -266,7 +287,7 @@ def test_status_file_initial_load(self): self.runtime.status_handler.set_patch_metadata_for_healthstore_substatus_json() self.runtime.execution_config.maintenance_run_id = str(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")) status_handler = StatusHandler(self.runtime.env_layer, self.runtime.execution_config, self.runtime.composite_logger, self.runtime.telemetry_writer, self.runtime.vm_cloud_type) - with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: substatus_file_data = json.load(file_handle)[0]["status"]["substatus"] self.assertTrue(len(substatus_file_data) == 1) @@ -274,7 +295,7 @@ def test_status_file_initial_load(self): self.runtime.status_handler.set_installation_reboot_status(Constants.RebootStatus.COMPLETED) self.runtime.execution_config.maintenance_run_id = str(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")) status_handler = StatusHandler(self.runtime.env_layer, self.runtime.execution_config, self.runtime.composite_logger, self.runtime.telemetry_writer, self.runtime.vm_cloud_type) - with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: substatus_file_data = json.load(file_handle)[0]["status"]["substatus"][0] self.assertTrue(status_handler is not None) self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["shouldReportToHealthStore"], False) @@ -348,6 +369,30 @@ def test_sequence_number_changed_termination_auto_assess_only(self): self.assertTrue(formatted_message["errors"]["details"][0]["code"] == Constants.PatchOperationErrorCodes.NEWER_OPERATION_SUPERSEDED) self.assertEqual(formatted_message["startedBy"], Constants.PatchAssessmentSummaryStartedBy.PLATFORM) + def test_sequence_number_changed_termination_configuration_only(self): + self.runtime.execution_config.operation = Constants.CONFIGURE_PATCHING + self.runtime.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) + + self.runtime.status_handler.report_sequence_number_changed_termination() + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0]["status"]["substatus"][0] + self.assertTrue(substatus_file_data["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_ERROR.lower()) + formatted_message = json.loads(substatus_file_data['formattedMessage']['message']) + self.assertTrue(formatted_message["errors"]["details"][0]["code"] == Constants.PatchOperationErrorCodes.NEWER_OPERATION_SUPERSEDED) + + def test_sequence_number_changed_termination_installation(self): + self.runtime.execution_config.operation = Constants.INSTALLATION + self.runtime.status_handler.set_current_operation(Constants.INSTALLATION) + + self.runtime.status_handler.report_sequence_number_changed_termination() + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0]["status"]["substatus"][0] + self.assertTrue(substatus_file_data["name"] == Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_ERROR.lower()) + formatted_message = json.loads(substatus_file_data['formattedMessage']['message']) + self.assertTrue(formatted_message["errors"]["details"][0]["code"] == Constants.PatchOperationErrorCodes.NEWER_OPERATION_SUPERSEDED) + def test_set_patch_metadata_for_healthstore_substatus_json_auto_assess_transitioning(self): self.runtime.execution_config.exec_auto_assess_only = True self.assertRaises(Exception, @@ -367,7 +412,6 @@ def test_sort_packages_by_classification_and_state(self): with self.runtime.env_layer.file_system.open("../../extension/tests/helpers/PatchOrderAssessmentSummary.json", 'r') as file_handle: assessment_patches = json.load(file_handle)["patches"] assessment_patches_sorted = self.runtime.status_handler.sort_packages_by_classification_and_state(assessment_patches) - print('what is assessment_patches', assessment_patches_sorted) # + Classifications | Patch State + # |--------------------|-------------| self.assertEqual(assessment_patches_sorted[0]["name"], "test-package-1") # | Critical | | @@ -436,6 +480,53 @@ def test_if_complete_and_status_path_is_dir(self): self.runtime.execution_config.complete_status_file_path = self.old_complete_status_path self.runtime.execution_config.status_file_path = self.old_status_path + def test_remove_old_complete_status_files(self): + """ Create dummy files in status folder and check if the complete_status_file_path is the latest file and delete those dummy files """ + # Set up create temp file for log and set sys.stdout to it + self.__create_temp_file_and_set_stdout() + + file_path = self.runtime.execution_config.status_folder + for i in range(1, 15): + with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f: + f.write("test" + str(i)) + + packages, package_versions = self.runtime.package_manager.get_all_updates() + self.runtime.status_handler.set_package_assessment_status(packages, package_versions) + self.runtime.status_handler.load_status_file_components(initial_load=True) + + # remove 10 complete status files + count_status_files = glob.glob(os.path.join(file_path, '*.complete.status')) + self.assertEqual(10, len(count_status_files)) + self.assertTrue(os.path.isfile(self.runtime.execution_config.complete_status_file_path)) + self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status') + self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status'))) + self.__read_temp_log_and_assert("Cleaned up older complete status files") + + # Reset sys.stdout, close and delete tmp + self.__remove_temp_file_reset_stdout() + + def test_remove_old_complete_status_files_throws_exception(self): + # Set up create temp file for log and set sys.stdout to it + self.__create_temp_file_and_set_stdout() + + file_path = self.runtime.execution_config.status_folder + for i in range(1, 16): + with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f: + f.write("test" + str(i)) + + self.backup_os_remove = os.remove + os.remove = self.__mock_os_remove + self.assertRaises(Exception, self.runtime.status_handler.load_status_file_components(initial_load=True)) + self.__read_temp_log_and_assert("Error deleting complete status file") + + # reset os.remove() mock and remove *complete.status files + os.remove = self.backup_os_remove + self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status') + self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status'))) + + # Reset sys.stdout, close and delete tmp + self.__remove_temp_file_reset_stdout() + def test_assessment_packages_map(self): patch_count_for_test = 5 expected_patch_id = 'python-samba0_2:4.4.5+dfsg-2ubuntu5.4_Ubuntu_16.04' @@ -535,38 +626,467 @@ def test_load_status_and_set_package_install_status(self): self.assertTrue('Critical' in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][2]["classifications"])) self.runtime.env_layer.file_system.delete_files_from_dir(self.runtime.status_handler.status_file_path, '*.complete.status') - def test_remove_old_complete_status_files(self): - """ Create dummy files in status folder and check if the complete_status_file_path is the latest file and delete those dummy files """ - file_path = self.runtime.execution_config.status_folder - for i in range(1, 15): - with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f: - f.write("test" + str(i)) + def test_log_truncated_packages_assert_no_truncation(self): + # Set up create temp file for log and set sys.stdout to it + self.__create_temp_file_and_set_stdout() - packages, package_versions = self.runtime.package_manager.get_all_updates() - self.runtime.status_handler.set_package_assessment_status(packages, package_versions) - self.runtime.status_handler.load_status_file_components(initial_load=True) + # Assert no truncation log output + patch_count_for_test = 500 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + self.runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) - # remove 10 complete status files - count_status_files = glob.glob(os.path.join(file_path, '*.complete.status')) - self.assertEqual(10, len(count_status_files)) - self.assertTrue(os.path.isfile(self.runtime.execution_config.complete_status_file_path)) - self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status') - self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status'))) + self.runtime.status_handler.log_truncated_packages() + self.__read_temp_log_and_assert("No packages truncated") - def test_remove_old_complete_status_files_throws_exception(self): - file_path = self.runtime.execution_config.status_folder - for i in range(1, 16): - with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f: - f.write("test" + str(i)) + # Reset sys.stdout, close and delete tmp + self.__remove_temp_file_reset_stdout() - self.backup_os_remove = os.remove - os.remove = self.__mock_os_remove - self.assertRaises(Exception, self.runtime.status_handler.load_status_file_components(initial_load=True)) + def test_log_truncated_packages_assert_assessment_truncation(self): + # Set up create temp file for log and set sys.stdout to it + self.__create_temp_file_and_set_stdout() + Constants.StatusTruncationConfig.NO_TRUNCATION_IN_X_SEC = -1 - # reset os.remove() mock and remove *complete.status files - os.remove = self.backup_os_remove - self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status') - self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status'))) + # Assert assessment truncation log output + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + self.runtime.status_handler.log_truncated_packages() + self.__read_temp_log_and_assert("Packages removed from assessment packages list") + + # Reset sys.stdout, close and delete tmp + self.__remove_temp_file_reset_stdout() + + def test_log_truncated_packages_assert_installation_truncation(self): + # Set up create temp file for log and set sys.stdout to it + self.__create_temp_file_and_set_stdout() + Constants.StatusTruncationConfig.NO_TRUNCATION_IN_X_SEC = -1 + + # Assert installation truncation log output + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + self.runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + self.runtime.status_handler.log_truncated_packages() + self.__read_temp_log_and_assert("Packages removed from installation packages list") + + # Reset sys.stdout, close and delete tmp + self.__remove_temp_file_reset_stdout() + + def test_assessment_status_file_truncation_under_size_limit(self): + self.runtime.execution_config.operation = Constants.ASSESSMENT + self.runtime.status_handler.set_current_operation(Constants.ASSESSMENT) + + patch_count_for_test = 500 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + self.runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.INTERNAL_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.ASSESSMENT) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.INTERNAL_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertNotEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + status_file_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertNotEqual(status_file_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertTrue('Truncated_patch_list_id' not in status_file_patches[-1]['name']) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], 0) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + self.assertFalse("review this log file on the machine" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + self.assertEqual(len(self.runtime.status_handler._StatusHandler__assessment_packages_removed), 0) + + def test_assessment_status_file_truncation_over_size_limit(self): + """ Test truncation logic will apply to assessment when it is over the size limit """ + self.runtime.execution_config.operation = Constants.ASSESSMENT + self.runtime.status_handler.set_current_operation(Constants.ASSESSMENT) + + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + self.runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.INTERNAL_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.ASSESSMENT) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertTrue("additional updates of classification" in message_patches[-1]['name'][0]) + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_assessment_status_file_truncation_over_large_size_limit_for_extra_chars(self): + """ Test truncation logic will apply to assessment, the 2 times json.dumps() will escape " adding \, adding 1 additional byte check if total byte size over the size limit """ + self.runtime.execution_config.operation = Constants.ASSESSMENT + self.runtime.status_handler.set_current_operation(Constants.ASSESSMENT) + + patch_count_for_test = 100000 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions, "Critical") + self.runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.ASSESSMENT) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertTrue("additional updates of classification" in message_patches[-1]['name'][0]) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue("review this log file on the machine" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_assessment_status_file_truncation_over_size_limit_with_errors(self): + """ Test truncation logic will apply to assessment with errors over the size limit """ + self.runtime.execution_config.operation = Constants.ASSESSMENT + self.runtime.status_handler.set_current_operation(Constants.ASSESSMENT) + + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions, "Security") + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0]["status"]["substatus"][0] + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + self.runtime.status_handler.set_assessment_substatus_json(status=Constants.STATUS_ERROR) + + # Adding multiple exceptions + self.runtime.status_handler.add_error_to_status("exception1", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception2", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception3", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception4", Constants.PatchOperationErrorCodes.PACKAGE_MANAGER_FAILURE) + self.runtime.status_handler.add_error_to_status("exception5", Constants.PatchOperationErrorCodes.OPERATION_FAILED) + + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertTrue(substatus_file_data["status"] != Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertNotEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"], None) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.ERROR) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 5) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.OPERATION_FAILED) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][1]["code"], Constants.PatchOperationErrorCodes.PACKAGE_MANAGER_FAILURE) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["message"], "exception5") + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_ASSESSMENT_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_ERROR.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertTrue("additional updates of classification" in message_patches[-1]['name'][0]) + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.ERROR) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 5) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue('6 error/s reported. The latest 5 error/s are shared in detail.' in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_installation_status_file_truncation_over_size_limit(self): + """ Test truncation logic will apply to installation over the size limit """ + self.runtime.execution_config.operation = Constants.INSTALLATION + self.runtime.status_handler.set_current_operation(Constants.INSTALLATION) + + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + self.runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-1]['name'], "Truncated_patch_list") + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue('1 error/s reported. The latest 1 error/s are shared in detail.' in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_installation_status_file_truncation_over_size_limit_low_priority_packages(self): + """ Test truncation logic will apply to installation over the size limit """ + self.runtime.execution_config.operation = Constants.INSTALLATION + self.runtime.status_handler.set_current_operation(Constants.INSTALLATION) + + patch_count_for_test = random.randint(780, 1100) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.PENDING) + self.runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-1]['name'], "Truncated_patch_list") + self.assertEqual(message_patches[-1]['patchInstallationState'], 'NotSelected') + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue('1 error/s reported. The latest 1 error/s are shared in detail.' in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_installation_status_file_truncation_over_large_size_limit_with_extra_chars(self): + """ Test truncation logic will apply to installation, the 2 times json.dumps() will escape " adding \, adding 1 additional byte check if total byte size over the size limit """ + self.runtime.execution_config.operation = Constants.INSTALLATION + self.runtime.status_handler.set_current_operation(Constants.INSTALLATION) + + patch_count_for_test = 100000 + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + self.runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_SUCCESS) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + self.assertEqual(substatus_file_data[0]["status"]["operation"], Constants.INSTALLATION) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_SUCCESS.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_WARNING.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-1]['name'], "Truncated_patch_list") + self.assertEqual(message_patches[-1]['patchInstallationState'], 'NotSelected') + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.WARNING) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 1) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + self.assertTrue('1 error/s reported. The latest 1 error/s are shared in detail.' in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_installation_status_file_truncation_over_size_limit_with_error(self): + """ Test truncation logic will apply to installation with errors over the size limit """ + self.runtime.execution_config.operation = Constants.INSTALLATION + self.runtime.status_handler.set_current_operation(Constants.INSTALLATION) + + patch_count_for_test = random.randint(780, 1000) + test_packages, test_package_versions = self.__set_up_packages_func(patch_count_for_test) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + + # Test Complete status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle)[0]["status"]["substatus"][0] + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 0) + + self.runtime.status_handler.set_installation_substatus_json(status=Constants.STATUS_ERROR) + + # Adding multiple exceptions + self.runtime.status_handler.add_error_to_status("exception0", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception1", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception2", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception3", Constants.PatchOperationErrorCodes.DEFAULT_ERROR) + self.runtime.status_handler.add_error_to_status("exception4", Constants.PatchOperationErrorCodes.PACKAGE_MANAGER_FAILURE) + self.runtime.status_handler.add_error_to_status("exception5", Constants.PatchOperationErrorCodes.OPERATION_FAILED) + + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.complete_status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) > Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertTrue(substatus_file_data["status"] == Constants.STATUS_ERROR.lower()) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"]), patch_count_for_test) + self.assertNotEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"], None) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], 1) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 5) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.OPERATION_FAILED) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][1]["code"], Constants.PatchOperationErrorCodes.PACKAGE_MANAGER_FAILURE) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["message"], "exception5") + + # Test Truncated status file + with self.runtime.env_layer.file_system.open(self.runtime.execution_config.status_file_path, 'r') as file_handle: + substatus_file_data = json.load(file_handle) + + self.assertTrue(len(json.dumps(substatus_file_data)) < Constants.StatusTruncationConfig.AGENT_FACING_STATUS_FILE_SIZE_LIMIT_IN_BYTES) + substatus_file_data = substatus_file_data[0]["status"]["substatus"][0] + self.assertEqual(substatus_file_data["name"], Constants.PATCH_INSTALLATION_SUMMARY) + self.assertEqual(substatus_file_data["status"], Constants.STATUS_ERROR.lower()) + message_patches = json.loads(substatus_file_data["formattedMessage"]["message"])["patches"] + self.assertEqual(message_patches[-1]['patchId'], "Truncated_patch_list_id") + self.assertEqual(message_patches[-1]['name'], "Truncated_patch_list") + self.assertTrue(len(message_patches) < patch_count_for_test + 1) # 1 tombstone + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["code"], Constants.PatchOperationTopLevelErrorCode.ERROR) + self.assertEqual(len(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"]), 5) + self.assertEqual(json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["details"][0]["code"], Constants.PatchOperationErrorCodes.TRUNCATION) + # 1 truncation error + self.assertTrue("7 error/s reported. The latest 5 error/s are shared in detail" in json.loads(substatus_file_data["formattedMessage"]["message"])["errors"]["message"]) + + def test_truncation_method_time_performance(self): + self.runtime.execution_config.operation = Constants.INSTALLATION + self.runtime.status_handler.set_current_operation(Constants.INSTALLATION) + self.__create_temp_file_and_set_stdout() # set tmp file for storing sys.stout() + + # Start no truncation performance test + Constants.StatusTruncationConfig.TURN_ON_TRUNCATION = False + no_truncate_start_time = time.time() + for i in range(0, 301): + test_packages, test_package_versions = self.__set_up_packages_func(500) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + + no_truncate_end_time = time.time() + no_truncate_performance_time = no_truncate_end_time - no_truncate_start_time + no_truncate_performance_time_formatted = self.__convert_test_performance_to_date_time(no_truncate_performance_time) + + # Start truncation performance test + Constants.StatusTruncationConfig.TURN_ON_TRUNCATION = True + truncate_start_time = time.time() + for i in range(0, 301): + test_packages, test_package_versions = self.__set_up_packages_func(500) + self.runtime.status_handler.set_package_assessment_status(test_packages, test_package_versions) + self.runtime.status_handler.set_package_install_status(test_packages, test_package_versions, Constants.INSTALLED) + + truncate_end_time = time.time() + truncate_performance_time = truncate_end_time - truncate_start_time + truncate_performance_time_formatted = self.__convert_test_performance_to_date_time(truncate_performance_time) + + self.__remove_temp_file_reset_stdout() # remove and reset tmp file for storing sys.stout() + + self.runtime.status_handler.composite_logger.log_debug('no_truncate_performance_time_formatted' + no_truncate_performance_time_formatted) + self.runtime.status_handler.composite_logger.log_debug('truncate_performance_time_formatted' + truncate_performance_time_formatted) + self.assertTrue(no_truncate_performance_time < truncate_performance_time) + + + # Setup functions for truncation + def __convert_test_performance_to_date_time(self, performance_time): + performance_time = abs(performance_time) + + # Calc days, hours, minutes, and seconds + days, remainder = divmod(performance_time, 86400) # 86400 seconds in a day + hours, remainder = divmod(remainder, 3600) # 3600 seconds in an hour + minutes, seconds = divmod(remainder, 60) # 60 seconds in a minute + + # Format the result + formatted_time = "%d days, %d hours, %d minutes, %.6f seconds" % (int(days), int(hours), int(minutes), seconds) + return formatted_time + + # Setup functions for writing log to temp and read output + def __create_temp_file_and_set_stdout(self): + # Set up create temp file for log and set sys.stdout to it + self.temp_stdout = tempfile.NamedTemporaryFile(delete=False, mode="w+") + self.saved_stdout = sys.stdout # Save the original stdout + sys.stdout = self.temp_stdout # set it to the temporary file + + def __remove_temp_file_reset_stdout(self): + sys.stdout = self.saved_stdout # redirect to original stdout + self.temp_stdout.close() + os.remove(self.temp_stdout.name) # Remove the temporary file + + def __read_temp_log_and_assert(self, expected_string): + self.temp_stdout.flush() + with open(self.temp_stdout.name, 'r') as temp_file: + captured_log_output = temp_file.read() + self.assertIn(expected_string, captured_log_output) # Setup functions to populate packages and versions for truncation def __set_up_packages_func(self, val):