From 7d90cffced03a33ac7a8f8572b353980d259ef72 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Tue, 31 Oct 2023 16:53:45 -0700 Subject: [PATCH 1/2] Reduced set: Safe-deployment enhancements (#223) * Reduced set - safe-deployment enhancements * Informational message requested added. * Increased test coverage * Test coverage and retry exhaustion throw * Minor comments --- src/core/src/bootstrap/Constants.py | 2 + src/core/src/core_logic/ExecutionConfig.py | 13 +++ src/core/src/core_logic/PatchInstaller.py | 96 ++++++++++++++--- .../AptitudePackageManager.py | 101 ++++++++++++++---- .../src/package_managers/PackageManager.py | 10 ++ .../src/package_managers/YumPackageManager.py | 6 ++ .../package_managers/ZypperPackageManager.py | 6 ++ .../src/service_interfaces/StatusHandler.py | 9 +- .../tests/Test_ConfigurePatchingProcessor.py | 3 +- src/core/tests/Test_CoreMain.py | 41 ++++--- src/core/tests/Test_PatchInstaller.py | 55 ++++++++-- .../tests/library/LegacyEnvLayerExtensions.py | 13 ++- src/core/tests/library/RuntimeCompositor.py | 7 ++ src/extension/src/Constants.py | 1 + src/extension/src/ProcessHandler.py | 4 +- .../file_handlers/ExtConfigSettingsHandler.py | 13 +-- .../tests/Test_ExtConfigSettingsHandler.py | 2 +- src/extension/tests/Test_ProcessHandler.py | 12 +++ src/extension/tests/helpers/1234.settings | 7 +- 19 files changed, 320 insertions(+), 81 deletions(-) diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index 88a8be54..c84d69fa 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -270,6 +270,7 @@ class PatchOperationTopLevelErrorCode(EnumBackport): ERROR = 1 class PatchOperationErrorCodes(EnumBackport): + INFORMATIONAL = "INFORMATIONAL" DEFAULT_ERROR = "ERROR" # default error code OPERATION_FAILED = "OPERATION_FAILED" PACKAGE_MANAGER_FAILURE = "PACKAGE_MANAGER_FAILURE" @@ -310,6 +311,7 @@ class TelemetryTaskName(EnumBackport): TELEMETRY_NOT_COMPATIBLE_ERROR_MSG = "Unsupported older Azure Linux Agent version. To resolve: http://aka.ms/UpdateLinuxAgent" TELEMETRY_COMPATIBLE_MSG = "Minimum Azure Linux Agent version prerequisite met" PYTHON_NOT_COMPATIBLE_ERROR_MSG = "Unsupported older Python version. Minimum Python version required is 2.7. [DetectedPythonVersion={0}]" + INFO_STRICT_SDP_SUCCESS = "Success: Safely patched your VM in a AzGPS-coordinated global rollout. https://aka.ms/AzGPS/StrictSDP [Target={0}]" UTC_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # EnvLayer Constants diff --git a/src/core/src/core_logic/ExecutionConfig.py b/src/core/src/core_logic/ExecutionConfig.py index 48044e67..6723a657 100644 --- a/src/core/src/core_logic/ExecutionConfig.py +++ b/src/core/src/core_logic/ExecutionConfig.py @@ -59,6 +59,7 @@ def __init__(self, env_layer, composite_logger, execution_parameters): self.excluded_package_name_mask_list = self.__get_execution_configuration_value_safely(self.config_settings, Constants.ConfigSettings.PATCHES_TO_EXCLUDE, []) self.maintenance_run_id = self.__get_execution_configuration_value_safely(self.config_settings, Constants.ConfigSettings.MAINTENANCE_RUN_ID) self.health_store_id = self.__get_execution_configuration_value_safely(self.config_settings, Constants.ConfigSettings.HEALTH_STORE_ID) + self.max_patch_publish_date = self.__get_max_patch_publish_date(self.health_store_id) if self.operation == Constants.INSTALLATION: self.reboot_setting = self.config_settings[Constants.ConfigSettings.REBOOT_SETTING] # expected to throw if not present else: @@ -99,6 +100,18 @@ def __transform_execution_config_for_auto_assessment(self): self.patch_mode = None self.composite_logger.log_debug("Setting execution configuration values for auto assessment. [GeneratedActivityId={0}][StartTime={1}]".format(self.activity_id, str(self.start_time))) + def __get_max_patch_publish_date(self, health_store_id): + # type: (str) -> object + """ Obtains implicit date ceiling for published date - converts pub_off_sku_2024.04.01 to 20240401T000000Z """ + max_patch_publish_date = str() + if health_store_id is not None and health_store_id != "": + split = health_store_id.split("_") + if len(split) == 4 and len(split[3]) == 10: + max_patch_publish_date = "{0}T000000Z".format(split[3].replace(".", "")) + + self.composite_logger.log_debug("[EC] Getting max patch publish date. [MaxPatchPublishDate={0}][HealthStoreId={1}]".format(str(max_patch_publish_date), str(health_store_id))) + return max_patch_publish_date + @staticmethod def __get_value_from_argv(argv, key, default_value=Constants.DEFAULT_UNSPECIFIED_VALUE): """ Discovers the value associated with a specific parameter in input arguments. """ diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index a80c1436..b7f4f4f6 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -81,8 +81,20 @@ def start_installation(self, simulate=False): self.composite_logger.log_debug("Attempting to reboot the machine prior to patch installation as there is a reboot pending...") reboot_manager.start_reboot_if_required_and_time_available(maintenance_window.get_remaining_time_in_minutes(None, False)) - # Install Updates - installed_update_count, update_run_successful, maintenance_window_exceeded = self.install_updates(maintenance_window, package_manager, simulate) + if self.execution_config.max_patch_publish_date != str(): + self.package_manager.set_max_patch_publish_date(self.execution_config.max_patch_publish_date) + + if self.package_manager.max_patch_publish_date != str(): + """ Strict SDP with the package manager that supports it """ + installed_update_count, update_run_successful, maintenance_window_exceeded = self.install_updates_azgps_coordinated(maintenance_window, package_manager, simulate) + package_manager.set_package_manager_setting(Constants.PACKAGE_MGR_SETTING_REPEAT_PATCH_OPERATION, bool(not update_run_successful)) + if update_run_successful: + self.composite_logger.log_debug(Constants.INFO_STRICT_SDP_SUCCESS.format(self.execution_config.max_patch_publish_date)) + self.status_handler.add_error_to_status(Constants.INFO_STRICT_SDP_SUCCESS.format(self.execution_config.max_patch_publish_date), error_code=Constants.PatchOperationErrorCodes.INFORMATIONAL) + else: + """ Regular patch installation flow - non-AzGPS-coordinated and (AzGPS-coordinated without strict SDP)""" + installed_update_count, update_run_successful, maintenance_window_exceeded = self.install_updates(maintenance_window, package_manager, simulate) + retry_count = 1 # Repeat patch installation if flagged as required and time is available if not maintenance_window_exceeded and package_manager.get_package_manager_setting(Constants.PACKAGE_MGR_SETTING_REPEAT_PATCH_OPERATION, False): @@ -148,6 +160,66 @@ def raise_if_min_python_version_not_met(self): self.status_handler.set_installation_substatus_json(status=Constants.STATUS_ERROR) raise Exception(error_msg) + def install_updates_azgps_coordinated(self, maintenance_window, package_manager, simulate=False): + """ Special-casing installation as it meets the following criteria: + - Maintenance window is always guaranteed to be nearly 4 hours (235 minutes). Customer-facing maintenance windows are much larger (system limitation). + - Barring reboot, the core Azure customer-base moving to coordinated, unattended upgrades is currently on a 24x7 MW. + - Built in service-level retries and management of outcomes. Reboot will only happen within the core maintenance window (and won't be delayed). + - Corner-case transient failures are immaterial to the overall functioning of AzGPS coordinated upgrades (eventual consistency). + - Only security updates (no other configuration) - simplistic execution flow; no advanced evaluation is desired or necessary. + """ + installed_update_count = 0 # includes dependencies + patch_installation_successful = True + maintenance_window_exceeded = False + remaining_time = maintenance_window.get_remaining_time_in_minutes() + + try: + all_packages, all_package_versions = package_manager.get_all_updates(True) + packages, package_versions = package_manager.get_security_updates() + self.last_still_needed_packages = list(all_packages) + self.last_still_needed_package_versions = list(all_package_versions) + + not_included_packages, not_included_package_versions = self.get_not_included_updates(package_manager, packages) + packages, package_versions, self.skipped_esm_packages, self.skipped_esm_package_versions, self.esm_packages_found_without_attach = package_manager.separate_out_esm_packages(packages, package_versions) + + self.status_handler.set_package_install_status(not_included_packages, not_included_package_versions, Constants.NOT_SELECTED) + self.status_handler.set_package_install_status(packages, package_versions, Constants.PENDING) + self.status_handler.set_package_install_status(self.skipped_esm_packages, self.skipped_esm_package_versions, Constants.FAILED) + + self.status_handler.set_package_install_status_classification(packages, package_versions, classification="Security") + package_manager.set_security_esm_package_status(Constants.INSTALLATION, packages) + + installed_update_count = 0 # includes dependencies + patch_installation_successful = True + maintenance_window_exceeded = False + + install_result = Constants.FAILED + for i in range(0, Constants.MAX_INSTALLATION_RETRY_COUNT): + code, out = package_manager.install_security_updates_azgps_coordinated() + installed_update_count += self.perform_status_reconciliation_conditionally(package_manager) + + remaining_time = maintenance_window.get_remaining_time_in_minutes() + if remaining_time < 120: + raise Exception("Not enough safety-buffer to continue strict safe deployment.") + + if code != 0: # will need to be modified for other package managers + if i < Constants.MAX_INSTALLATION_RETRY_COUNT - 1: + time.sleep(i * 5) + self.composite_logger.log_warning("[PI][AzGPS-Coordinated] Non-zero return. Retrying. [RetryCount={0}][TimeRemainingInMins={1}][Code={2}][Output={3}]".format(str(i), str(remaining_time), str(code), out)) + else: + raise Exception("AzGPS Strict SDP retries exhausted. [RetryCount={0}]".format(str(i))) + else: + patch_installation_successful = True + break + except Exception as error: + error_msg = "AzGPS strict safe deployment to target date hit a failure. Defaulting to regular upgrades. [MaxPatchPublishDate={0}]".format(self.execution_config.max_patch_publish_date) + self.composite_logger.log_error(error_msg + "[Error={0}]".format(repr(error))) + self.status_handler.add_error_to_status(error_msg) + self.package_manager.set_max_patch_publish_date() # fall-back + patch_installation_successful = False + + return installed_update_count, patch_installation_successful, maintenance_window_exceeded + def install_updates(self, maintenance_window, package_manager, simulate=False): """wrapper function of installing updates""" self.composite_logger.log("\n\nGetting available updates...") @@ -576,20 +648,10 @@ def mark_installation_completed(self): self.status_handler.set_installation_substatus_json(status=Constants.STATUS_WARNING) # Update patch metadata in status for auto patching request, to be reported to healthStore - # When available, HealthStoreId always takes precedence over the 'overriden' Maintenance Run Id that is being re-purposed for other reasons - # In the future, maintenance run id will be completely deprecated for health store reporting. - patch_version_raw = self.execution_config.health_store_id if self.execution_config.health_store_id is not None else self.execution_config.maintenance_run_id - self.composite_logger.log_debug("Patch version raw value set. [Raw={0}][HealthStoreId={1}][MaintenanceRunId={2}]".format(str(patch_version_raw), str(self.execution_config.health_store_id), str(self.execution_config.maintenance_run_id))) - - if patch_version_raw is not None: - try: - patch_version = datetime.datetime.strptime(patch_version_raw.split(" ")[0], "%m/%d/%Y").strftime('%Y.%m.%d') - except ValueError as e: - patch_version = str(patch_version_raw) # CRP is supposed to guarantee that healthStoreId is always in the correct format; (Legacy) Maintenance Run Id may not be; what happens prior to this is just defensive coding - self.composite_logger.log_debug("Patch version _may_ be in an incorrect format. [CommonFormat=DateTimeUTC][Actual={0}][Error={1}]".format(str(self.execution_config.maintenance_run_id), repr(e))) - + self.composite_logger.log_debug("[PI] Reviewing final healthstore record write. [HealthStoreId={0}][MaintenanceRunId={1}]".format(str(self.execution_config.health_store_id), str(self.execution_config.maintenance_run_id))) + if self.execution_config.health_store_id is not None: self.status_handler.set_patch_metadata_for_healthstore_substatus_json( - patch_version=patch_version if patch_version is not None and patch_version != "" else Constants.PATCH_VERSION_UNKNOWN, + patch_version=self.execution_config.health_store_id, report_to_healthstore=True, wait_after_update=False) @@ -602,7 +664,7 @@ def perform_status_reconciliation_conditionally(self, package_manager, condition if not condition: return 0 - self.composite_logger.log_debug("\nStarting status reconciliation...") + self.composite_logger.log_verbose("\nStarting status reconciliation...") start_time = time.time() still_needed_packages, still_needed_package_versions = package_manager.get_all_updates(False) # do not use cache successful_packages = [] @@ -615,7 +677,7 @@ def perform_status_reconciliation_conditionally(self, package_manager, condition self.status_handler.set_package_install_status(successful_packages, successful_package_versions, Constants.INSTALLED) self.last_still_needed_packages = still_needed_packages self.last_still_needed_package_versions = still_needed_package_versions - self.composite_logger.log_debug("Completed status reconciliation. Time taken: " + str(time.time() - start_time) + " seconds.") + self.composite_logger.log_verbose("Completed status reconciliation. Time taken: " + str(time.time() - start_time) + " seconds.") return len(successful_packages) # endregion diff --git a/src/core/src/package_managers/AptitudePackageManager.py b/src/core/src/package_managers/AptitudePackageManager.py index 737de12b..c537a665 100644 --- a/src/core/src/package_managers/AptitudePackageManager.py +++ b/src/core/src/package_managers/AptitudePackageManager.py @@ -33,27 +33,28 @@ class AptitudePackageManager(PackageManager): def __init__(self, env_layer, execution_config, composite_logger, telemetry_writer, status_handler): super(AptitudePackageManager, self).__init__(env_layer, execution_config, composite_logger, telemetry_writer, status_handler) - security_list_guid = str(uuid.uuid4()) - # Accept EULA (End User License Agreement) as per the EULA settings set by user optional_accept_eula_in_cmd = "ACCEPT_EULA=Y" if execution_config.accept_package_eula else "" # Repo refresh - self.repo_refresh = 'sudo apt-get -q update' + self.cmd_repo_refresh_template = 'sudo apt-get -q update ' # Support to get updates and their dependencies - self.security_sources_list = os.path.join(execution_config.temp_folder, 'msft-patch-security-{0}.list'.format(security_list_guid)) - self.prep_security_sources_list_cmd = 'sudo grep -hR security /etc/apt/sources.list /etc/apt/sources.list.d/ > ' + os.path.normpath(self.security_sources_list) - self.dist_upgrade_simulation_cmd_template = 'LANG=en_US.UTF8 sudo apt-get -s dist-upgrade ' # Dist-upgrade simulation template - needs to be replaced before use; sudo is used as sometimes the sources list needs sudo to be readable - self.single_package_check_versions = 'apt-cache madison ' - self.single_package_find_installed_dpkg = 'sudo dpkg -s ' - self.single_package_find_installed_apt = 'sudo apt list --installed ' - self.single_package_upgrade_simulation_cmd = '''DEBIAN_FRONTEND=noninteractive ''' + optional_accept_eula_in_cmd + ''' apt-get -y --only-upgrade true -s install ''' + self.cached_customer_source_list_formula = None + self.custom_sources_list = os.path.join(execution_config.temp_folder, 'azgps-patch-custom-{0}.list'.format(str(uuid.uuid4()))) + self.cmd_prep_custom_sources_list_template = 'sudo grep -hR /etc/apt/sources.list /etc/apt/sources.list.d/ > ' + os.path.normpath(self.custom_sources_list) + self.cmd_dist_upgrade_simulation_template = 'LANG=en_US.UTF8 sudo apt-get -s dist-upgrade ' # Dist-upgrade simulation template - needs to be replaced before use; sudo is used as sometimes the sources list needs sudo to be readable + + self.cmd_single_package_check_versions_template = 'apt-cache madison ' + self.cmd_single_package_find_install_dpkg_template = 'sudo dpkg -s ' + self.cmd_single_package_find_install_apt_template = 'sudo apt list --installed ' + self.single_package_upgrade_simulation_cmd = '''DEBIAN_FRONTEND=noninteractive ''' + optional_accept_eula_in_cmd + ''' LANG=en_US.UTF8 apt-get -y --only-upgrade true -s install ''' self.single_package_dependency_resolution_template = 'DEBIAN_FRONTEND=noninteractive ' + optional_accept_eula_in_cmd + ' LANG=en_US.UTF8 apt-get -y --only-upgrade true -s install ' # Install update # --only-upgrade: upgrade only single package (only if it is installed) - self.single_package_upgrade_cmd = '''sudo DEBIAN_FRONTEND=noninteractive ''' + optional_accept_eula_in_cmd + ''' apt-get -y --only-upgrade true install ''' + self.single_package_upgrade_cmd = '''sudo DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF8 ''' + optional_accept_eula_in_cmd + ''' apt-get -y --only-upgrade true install ''' + self.install_security_updates_azgps_coordinated_cmd = '''sudo DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF8 ''' + optional_accept_eula_in_cmd + ''' apt-get -y --only-upgrade true upgrade ''' # Package manager exit code(s) self.apt_exitcode_ok = 0 @@ -79,9 +80,58 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.ubuntu_pro_client_all_updates_cached = [] self.ubuntu_pro_client_all_updates_versions_cached = [] - def refresh_repo(self): - self.composite_logger.log("\nRefreshing local repo...") - self.invoke_package_manager(self.repo_refresh) + # region Sources Management + def __get_custom_sources_to_spec(self, max_patch_published_date=str(), base_classification=str()): + # type: (str, str) -> str + """ Prepares the custom sources list for use in a command. Idempotent. """ + try: + if max_patch_published_date != str() and len(max_patch_published_date) != 16: + raise Exception("[APM] Invalid max patch published date received. [Value={0}]".format(str(max_patch_published_date))) + + formula = "F-[{0}]-[{1}]".format(max_patch_published_date, base_classification) + if self.cached_customer_source_list_formula == formula: + return self.custom_sources_list + self.cached_customer_source_list_formula = formula + + command = self.cmd_prep_custom_sources_list_template.replace("", base_classification if base_classification == "security" else "\"\"") + code, out = self.env_layer.run_command_output(command, False, False) + sources_content = self.env_layer.file_system.read_with_retry(self.custom_sources_list) + self.composite_logger.log_debug("[APM] Modified custom sources list with classification. [Code={0}][Out={1}][Content={2}]".format(str(code), str(out), str(sources_content))) # non-zero error code to be investigated + + if max_patch_published_date != str(): + target = "://snapshot.ubuntu.com/ubuntu/{0}".format(self.max_patch_publish_date) + + if "://snapshot.ubuntu.com/ubuntu/" in sources_content: + sources_content_split = sources_content.split(" ") + for i in range(0, len(sources_content_split)): + if "://snapshot.ubuntu.com/ubuntu/" in sources_content_split[i]: + sources_content.replace(sources_content_split[i], target) + + sources_content = sources_content.replace("://azure.archive.ubuntu.com/ubuntu/", target) + sources_content = sources_content.replace("://in.archive.ubuntu.com/ubuntu/", target) + sources_content = sources_content.replace("://security.ubuntu.com/ubuntu/", target) + sources_content = sources_content.replace("http://snapshot.ubuntu.com/", "https://snapshot.ubuntu.com/") + self.composite_logger.log_debug("[APM] Modified custom sources list with snapshot. [Code={0}][Out={1}][Content={2}]".format(str(code), str(out), str(sources_content))) + + self.env_layer.file_system.write_with_retry_using_temp_file(self.custom_sources_list, sources_content) + + self.refresh_repo(sources=self.custom_sources_list) + except Exception as error: + self.composite_logger.log_error("[APM] Error in modifying custom sources list. [Error={0}]".format(repr(error))) + return str() # defaults code to safety + + return self.custom_sources_list + + def refresh_repo(self, sources=str()): + self.composite_logger.log("[APM] Refreshing local repo... [Sources={0}]".format(sources if sources != str() else "Default")) + self.invoke_package_manager(self.__generate_command(self.cmd_repo_refresh_template, sources)) + + @staticmethod + def __generate_command(command_template, new_sources_list=str()): + # type: (str, str) -> str + """ Prepares a standard command to use custom sources. Pre-requisite: Refresh repo post list change. """ + return command_template.replace('', ('-oDir::Etc::Sourcelist={0}'.format(str(new_sources_list))) if new_sources_list != str() else str()) + # endregion Sources Management # region Get Available Updates def invoke_package_manager_advanced(self, command, raise_on_exception=True): @@ -157,7 +207,7 @@ def get_all_updates(self, cached=False): return all_updates, all_updates_versions # when cached is False, query both default way and using Ubuntu Pro Client. - cmd = self.dist_upgrade_simulation_cmd_template.replace('', '') + cmd = self.__generate_command(self.cmd_dist_upgrade_simulation_template, self.__get_custom_sources_to_spec(self.max_patch_publish_date)) out = self.invoke_package_manager(cmd) self.all_updates_cached, self.all_update_versions_cached = self.extract_packages_and_versions(out) @@ -183,11 +233,7 @@ def get_security_updates(self): ubuntu_pro_client_security_package_versions = [] self.composite_logger.log("\nDiscovering 'security' packages...") - code, out = self.env_layer.run_command_output(self.prep_security_sources_list_cmd, False, False) - if code != 0: - self.composite_logger.log_warning(" - SLP:: Return code: " + str(code) + ", Output: \n|\t" + "\n|\t".join(out.splitlines())) - - cmd = self.dist_upgrade_simulation_cmd_template.replace('', '-oDir::Etc::Sourcelist=' + self.security_sources_list) + cmd = self.__generate_command(self.cmd_dist_upgrade_simulation_template, self.__get_custom_sources_to_spec(self.max_patch_publish_date, base_classification="security")) out = self.invoke_package_manager(cmd) security_packages, security_package_versions = self.extract_packages_and_versions(out) @@ -239,6 +285,10 @@ def get_other_updates(self): return ubuntu_pro_client_other_packages, ubuntu_pro_client_other_package_versions else: return other_packages, other_package_versions + + def set_max_patch_publish_date(self, max_patch_publish_date=str()): + self.composite_logger.log_debug("[APM] Setting max patch publish date. [Date={0}]".format(max_patch_publish_date)) + self.max_patch_publish_date = max_patch_publish_date # endregion # region Output Parser(s) @@ -292,6 +342,11 @@ def get_composite_package_identifier(self, package, package_version): def install_updates_fail_safe(self, excluded_packages): return + + def install_security_updates_azgps_coordinated(self): + command = self.__generate_command(self.install_security_updates_azgps_coordinated_cmd, self.__get_custom_sources_to_spec(self.max_patch_publish_date, base_classification="security")) + out, code = self.invoke_package_manager_advanced(command, raise_on_exception=False) + return code, out # endregion # region Package Information @@ -304,7 +359,7 @@ def get_all_available_versions_of_package(self, package_name): package_versions = [] - cmd = self.single_package_check_versions.replace('', package_name) + cmd = self.cmd_single_package_check_versions_template.replace('', package_name) output = self.invoke_apt_cache(cmd) lines = output.strip().split('\n') @@ -325,7 +380,7 @@ def is_package_version_installed(self, package_name, package_version): # DEFAULT METHOD self.composite_logger.log_debug(" - [1/2] Verifying install status with Dpkg.") - cmd = self.single_package_find_installed_dpkg.replace('', package_name) + cmd = self.cmd_single_package_find_install_dpkg_template.replace('', package_name) code, output = self.env_layer.run_command_output(cmd, False, False) lines = output.strip().split('\n') @@ -396,7 +451,7 @@ def is_package_version_installed(self, package_name, package_version): # Listing... Done # apt/xenial-updates,now 1.2.29 amd64 [installed] self.composite_logger.log_debug(" - [2/2] Verifying install status with Apt.") - cmd = self.single_package_find_installed_apt.replace('', package_name) + cmd = self.cmd_single_package_find_install_apt_template.replace('', package_name) output = self.invoke_package_manager(cmd) lines = output.strip().split('\n') diff --git a/src/core/src/package_managers/PackageManager.py b/src/core/src/package_managers/PackageManager.py index 824fa999..3e89720a 100644 --- a/src/core/src/package_managers/PackageManager.py +++ b/src/core/src/package_managers/PackageManager.py @@ -42,6 +42,9 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ # auto OS updates self.image_default_patch_configuration_backup_path = os.path.join(execution_config.config_folder, Constants.IMAGE_DEFAULT_PATCH_CONFIGURATION_BACKUP_PATH) + # strict SDP + self.max_patch_publish_date = str() + # Constants self.STR_NOTHING_TO_DO = "Error: Nothing to do" self.STR_ONLY_UPGRADES = "Skipping , it is not installed and only upgrades are requested." @@ -108,6 +111,10 @@ def get_security_updates(self): @abstractmethod def get_other_updates(self): pass + + @abstractmethod + def set_max_patch_publish_date(self, max_patch_publish_date=str()): + pass # endregion def get_updates_for_inclusions(self, package_filter): @@ -329,6 +336,9 @@ def install_update_and_dependencies_and_get_status(self, package_and_dependencie install_result = self.get_installation_status(code, out, exec_cmd, package_and_dependencies[0], package_and_dependency_versions[0], simulate) return install_result + @abstractmethod + def install_security_updates_azgps_coordinated(self): + pass # endregion # region Package Information diff --git a/src/core/src/package_managers/YumPackageManager.py b/src/core/src/package_managers/YumPackageManager.py index a6ccceae..720be912 100644 --- a/src/core/src/package_managers/YumPackageManager.py +++ b/src/core/src/package_managers/YumPackageManager.py @@ -174,6 +174,9 @@ def get_other_updates(self): self.composite_logger.log("Discovered " + str(len(other_packages)) + " 'other' package entries.") return other_packages, other_package_versions + def set_max_patch_publish_date(self, max_patch_publish_date=str()): + pass + def install_yum_security_prerequisite(self): """Not installed by default in versions prior to RHEL 7. This step is idempotent and fast, so we're not writing more complex code.""" self.composite_logger.log_debug('Ensuring RHEL yum-plugin-security is present.') @@ -249,6 +252,9 @@ def install_updates_fail_safe(self, excluded_packages): self.composite_logger.log_debug("[FAIL SAFE MODE] UPDATING PACKAGES USING COMMAND: " + cmd) self.invoke_package_manager(cmd) + + def install_security_updates_azgps_coordinated(self): + pass # endregion # region Package Information diff --git a/src/core/src/package_managers/ZypperPackageManager.py b/src/core/src/package_managers/ZypperPackageManager.py index 5f172d40..70d901b4 100644 --- a/src/core/src/package_managers/ZypperPackageManager.py +++ b/src/core/src/package_managers/ZypperPackageManager.py @@ -344,6 +344,9 @@ def get_other_updates(self): self.composite_logger.log_debug("Discovered " + str(len(other_packages)) + " 'other' package entries.\n") return other_packages, other_package_versions + + def set_max_patch_publish_date(self, max_patch_publish_date=str()): + pass # endregion # region Output Parser(s) @@ -415,6 +418,9 @@ def get_composite_package_identifier(self, package, package_version): def install_updates_fail_safe(self, excluded_packages): return + + def install_security_updates_azgps_coordinated(self): + pass # endregion # region Package Information diff --git a/src/core/src/service_interfaces/StatusHandler.py b/src/core/src/service_interfaces/StatusHandler.py index 76c5c5ef..19f12b6d 100644 --- a/src/core/src/service_interfaces/StatusHandler.py +++ b/src/core/src/service_interfaces/StatusHandler.py @@ -800,8 +800,13 @@ def __try_add_error(error_list, detail): def __set_errors_json(self, error_count_by_operation, errors_by_operation): """ Compose the error object json to be added in 'errors' in given operation's summary """ - message = "{0} error/s reported.".format(error_count_by_operation) - message += " The latest {0} error/s are shared in detail. To view all errors, review this log file on the machine: {1}".format(len(errors_by_operation), self.__log_file_path) if error_count_by_operation > 0 else "" + if error_count_by_operation == 1 and errors_by_operation[0]['code'] == Constants.PatchOperationErrorCodes.INFORMATIONAL: # special-casing for single informational messages + message = errors_by_operation[0]['message'] + errors_by_operation = [] + error_count_by_operation = 0 + else: + message = "{0} error/s reported.".format(error_count_by_operation) + message += " The latest {0} error/s are shared in detail. To view all errors, review this log file on the machine: {1}".format(len(errors_by_operation), self.__log_file_path) if error_count_by_operation > 0 else "" return { "code": Constants.PatchOperationTopLevelErrorCode.SUCCESS if error_count_by_operation == 0 else Constants.PatchOperationTopLevelErrorCode.ERROR, "details": errors_by_operation, diff --git a/src/core/tests/Test_ConfigurePatchingProcessor.py b/src/core/tests/Test_ConfigurePatchingProcessor.py index f36c884c..e953a839 100644 --- a/src/core/tests/Test_ConfigurePatchingProcessor.py +++ b/src/core/tests/Test_ConfigurePatchingProcessor.py @@ -116,6 +116,7 @@ def test_operation_success_for_installation_request_with_configure_patching(self argument_composer = ArgumentComposer() argument_composer.operation = Constants.INSTALLATION argument_composer.maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" + argument_composer.health_store_id = "pub_off_sku_2020.09.23" argument_composer.patch_mode = Constants.PatchModes.AUTOMATIC_BY_PLATFORM runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) runtime.package_manager.get_current_auto_os_patch_state = runtime.backup_get_current_auto_os_patch_state @@ -158,7 +159,7 @@ def test_operation_success_for_installation_request_with_configure_patching(self self.assertTrue(substatus_file_data[2]["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) substatus_file_data_patch_metadata_summary = json.loads(substatus_file_data[2]["formattedMessage"]["message"]) - self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], "2020.09.28") + self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], "pub_off_sku_2020.09.23") self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) runtime.stop() diff --git a/src/core/tests/Test_CoreMain.py b/src/core/tests/Test_CoreMain.py index 922fa900..2d542e27 100644 --- a/src/core/tests/Test_CoreMain.py +++ b/src/core/tests/Test_CoreMain.py @@ -128,9 +128,12 @@ def test_operation_success_for_non_autopatching_request(self): def test_operation_success_for_autopatching_request(self): # test with valid datetime string for maintenance run id argument_composer = ArgumentComposer() - maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" + maintenance_run_id = "9/30/2020 02:00:00 PM +00:00" argument_composer.maintenance_run_id = str(maintenance_run_id) + argument_composer.health_store_id = str("2020.09.28") runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + Constants.MAX_FILE_OPERATION_RETRY_COUNT = 2 # to retain retry coverage in at least one test + Constants.MAX_IMDS_CONNECTION_RETRY_COUNT = 2 # to retain retry coverage in at least one test runtime.set_legacy_test_type('SuccessInstallPath') CoreMain(argument_composer.get_composed_arguments()) @@ -157,9 +160,10 @@ def test_operation_success_for_autopatching_request(self): def test_operation_success_for_autopatching_request_with_security_classification(self): # test with valid datetime string for maintenance run id argument_composer = ArgumentComposer() - maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" + maintenance_run_id = "9/30/2020 02:00:00 PM +00:00" classifications_to_include = ["Security", "Critical"] argument_composer.maintenance_run_id = str(maintenance_run_id) + argument_composer.health_store_id = str("2020.09.28") argument_composer.classifications_to_include = classifications_to_include runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) runtime.set_legacy_test_type("SuccessInstallPath") @@ -192,11 +196,11 @@ def test_operation_success_for_autopatching_request_with_security_classification self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) runtime.stop() - def test_invalid_maintenance_run_id(self): - # test with empty string for maintenence run id + def test_health_store_id_reporting(self): + # test with empty string for healthstoreid argument_composer = ArgumentComposer() - maintenance_run_id = "" - argument_composer.maintenance_run_id = maintenance_run_id + health_store_id = "pub_offer_sku_wrong_123" + argument_composer.health_store_id = health_store_id runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) runtime.set_legacy_test_type('SuccessInstallPath') CoreMain(argument_composer.get_composed_arguments()) @@ -215,16 +219,17 @@ def test_invalid_maintenance_run_id(self): self.assertTrue(substatus_file_data[2]["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) substatus_file_data_patch_metadata_summary = json.loads(substatus_file_data[2]["formattedMessage"]["message"]) - self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], Constants.PATCH_VERSION_UNKNOWN) + self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], health_store_id) + self.assertTrue(runtime.execution_config.max_patch_publish_date is str()) self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) runtime.stop() - # test with a random string for maintenance run id + # test with healthstoreid argument_composer = ArgumentComposer() - maintenance_run_id = "test" - argument_composer.maintenance_run_id = maintenance_run_id + health_store_id = "publ_off_sku_2024.04.01" + argument_composer.health_store_id = health_store_id runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) runtime.set_legacy_test_type('SuccessInstallPath') CoreMain(argument_composer.get_composed_arguments()) @@ -243,7 +248,8 @@ def test_invalid_maintenance_run_id(self): self.assertTrue(substatus_file_data[2]["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) substatus_file_data_patch_metadata_summary = json.loads(substatus_file_data[2]["formattedMessage"]["message"]) - self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], maintenance_run_id) + self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], health_store_id) + self.assertEqual(runtime.execution_config.max_patch_publish_date, "20240401T000000Z") self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) @@ -429,6 +435,7 @@ def test_install_all_packages_for_centos_autopatching(self): maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" classifications_to_include = ["Security", "Critical"] argument_composer.maintenance_run_id = str(maintenance_run_id) + argument_composer.health_store_id = str("pub_off_sku_2020.09.29") argument_composer.classifications_to_include = classifications_to_include argument_composer.reboot_setting = 'Always' runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) @@ -460,7 +467,7 @@ def test_install_all_packages_for_centos_autopatching(self): self.assertTrue(substatus_file_data[2]["name"] == Constants.PATCH_METADATA_FOR_HEALTHSTORE) self.assertTrue(substatus_file_data[2]["status"].lower() == Constants.STATUS_SUCCESS.lower()) substatus_file_data_patch_metadata_summary = json.loads(substatus_file_data[2]["formattedMessage"]["message"]) - self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], "2020.09.28") + self.assertEqual(substatus_file_data_patch_metadata_summary["patchVersion"], "pub_off_sku_2020.09.29") self.assertTrue(substatus_file_data_patch_metadata_summary["shouldReportToHealthStore"]) self.assertTrue(substatus_file_data[3]["name"] == Constants.CONFIGURE_PATCHING_SUMMARY) self.assertTrue(substatus_file_data[3]["status"].lower() == Constants.STATUS_SUCCESS.lower()) @@ -477,9 +484,10 @@ def test_install_all_packages_for_centos_autopatching_as_warning_with_never_rebo LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_centos argument_composer = ArgumentComposer() - maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" + maintenance_run_id = "9/30/2020 02:00:00 PM +00:00" classifications_to_include = ["Security", "Critical"] argument_composer.maintenance_run_id = str(maintenance_run_id) + argument_composer.health_store_id = str("2020.09.28") argument_composer.classifications_to_include = classifications_to_include runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) runtime.set_legacy_test_type("HappyPath") @@ -525,9 +533,10 @@ def test_install_only_critical_and_security_packages_for_redhat_autopatching(sel LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_redhat argument_composer = ArgumentComposer() - maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" + maintenance_run_id = "9/30/2020 02:00:00 PM +00:00" classifications_to_include = ["Security", "Critical"] argument_composer.maintenance_run_id = str(maintenance_run_id) + argument_composer.health_store_id = str("2020.09.28") argument_composer.classifications_to_include = classifications_to_include argument_composer.reboot_setting = 'Always' runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) @@ -577,14 +586,14 @@ def test_install_only_critical_and_security_packages_for_redhat_autopatching_war """Unit test for auto patching request on Redhat, should install only critical and security patches, installation status is set to warning when reboot_setting is never_reboot """ - backup_envlayer_platform_linux_distribution = LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution LegacyEnvLayerExtensions.LegacyPlatform.linux_distribution = self.mock_linux_distribution_to_return_redhat argument_composer = ArgumentComposer() - maintenance_run_id = "9/28/2020 02:00:00 PM +00:00" + maintenance_run_id = "9/30/2020 02:00:00 PM +00:00" classifications_to_include = ["Security", "Critical"] argument_composer.maintenance_run_id = str(maintenance_run_id) + argument_composer.health_store_id = str("2020.09.28") argument_composer.classifications_to_include = classifications_to_include runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) runtime.set_legacy_test_type("HappyPath") diff --git a/src/core/tests/Test_PatchInstaller.py b/src/core/tests/Test_PatchInstaller.py index 6d6a81e7..4d6ca663 100644 --- a/src/core/tests/Test_PatchInstaller.py +++ b/src/core/tests/Test_PatchInstaller.py @@ -16,6 +16,7 @@ import datetime import json +import os import sys import unittest from core.src.bootstrap.Constants import Constants @@ -229,6 +230,53 @@ def test_apt_install_skips_esm_packages(self): obj.mock_unimport_uaclient_update_module() version_obj.mock_unimport_uaclient_version_module() + def test_patch_installer_for_azgps_coordinated(self): + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = "PT235M" + argument_composer.health_store_id = "pub_offer_sku_2024.04.01" + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + runtime.package_manager.custom_sources_list = os.path.join(argument_composer.temp_folder, "temp2.list") + # Path change + runtime.set_legacy_test_type('HappyPath') + self.assertTrue(runtime.patch_installer.start_installation()) + self.assertEqual(runtime.execution_config.max_patch_publish_date, "20240401T000000Z") + self.assertEqual(runtime.package_manager.max_patch_publish_date,"20240401T000000Z") # supported and conditions met + runtime.stop() + + argument_composer.maximum_duration = "PT30M" + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + runtime.set_legacy_test_type('HappyPath') + self.assertFalse(runtime.patch_installer.start_installation()) # failure is in unrelated patch installation batch processing + self.assertEqual(runtime.execution_config.max_patch_publish_date, "20240401T000000Z") + self.assertEqual(runtime.package_manager.max_patch_publish_date, "") # reason: not enough time to use + + runtime.package_manager.max_patch_publish_date = "Wrong" + runtime.package_manager.get_security_updates() # exercises an exception path on bad data without throwing an exception (graceful degradation to security) + runtime.stop() + + argument_composer.maximum_duration = "PT235M" + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + runtime.set_legacy_test_type('HappyPath') + runtime.package_manager.install_security_updates_azgps_coordinated = lambda: (1, "Failed") + self.assertFalse(runtime.patch_installer.start_installation()) + self.assertEqual(runtime.execution_config.max_patch_publish_date, "20240401T000000Z") + self.assertEqual(runtime.package_manager.max_patch_publish_date, "") # reason: the strict SDP is forced to fail with the lambda above + runtime.stop() + + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + runtime.set_legacy_test_type('HappyPath') + self.assertTrue(runtime.patch_installer.start_installation()) + self.assertEqual(runtime.execution_config.max_patch_publish_date, "20240401T000000Z") + self.assertEqual(runtime.package_manager.max_patch_publish_date, "") # unsupported in Yum + runtime.stop() + + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + runtime.set_legacy_test_type('HappyPath') + self.assertFalse(runtime.patch_installer.start_installation()) # failure is in unrelated patch installation batch processing + self.assertEqual(runtime.execution_config.max_patch_publish_date, "20240401T000000Z") + self.assertEqual(runtime.package_manager.max_patch_publish_date, "") # unsupported in Zypper + runtime.stop() + def test_mark_status_completed_esm_required(self): obj = MockUpdatesResult() obj.mock_import_uaclient_update_module('updates', 'mock_update_list_with_one_esm_update') @@ -526,12 +574,7 @@ def test_no_updates_to_install(self): def test_healthstore_writes(self): self.healthstore_writes_helper("HealthStoreId", None, False, expected_patch_version="HealthStoreId") self.healthstore_writes_helper("HealthStoreId", "MaintenanceRunId", False, expected_patch_version="HealthStoreId") - self.healthstore_writes_helper(None, "MaintenanceRunId", False, expected_patch_version="MaintenanceRunId") - self.healthstore_writes_helper(None, "09/16/2021 08:24:42 AM +00:00", False, expected_patch_version="2021.09.16") - self.healthstore_writes_helper("09/16/2021 08:24:42 AM +00:00", None, False, expected_patch_version="2021.09.16") - self.healthstore_writes_helper("09/16/2021 08:24:42 AM +00:00", "09/17/2021 08:24:42 AM +00:00", False, expected_patch_version="2021.09.16") - self.healthstore_writes_helper("09/16/2021 08:24:42 AM +00:00", "", False, expected_patch_version="2021.09.16") - self.healthstore_writes_helper("09/16/2021 08:24:42 AM +00:00", "", True, expected_patch_version="2021.09.16") + self.healthstore_writes_helper("pub_offer_sku_2020.10.20", None, False, expected_patch_version="pub_offer_sku_2020.10.20") def healthstore_writes_helper(self, health_store_id, maintenance_run_id, is_force_reboot, expected_patch_version): current_time = datetime.datetime.utcnow() diff --git a/src/core/tests/library/LegacyEnvLayerExtensions.py b/src/core/tests/library/LegacyEnvLayerExtensions.py index d0316722..a3d34bfc 100644 --- a/src/core/tests/library/LegacyEnvLayerExtensions.py +++ b/src/core/tests/library/LegacyEnvLayerExtensions.py @@ -460,7 +460,14 @@ def run_command_output(self, cmd, no_output=False, chk_err=True): "Ubuntu:16.10/yakkety-security [amd64]) []\n" + \ "Inst samba-libs [2:4.4.5+dfsg-2ubuntu5.2] (2:4.4.5+dfsg-2ubuntu5.4 " + \ "Ubuntu:16.10/yakkety-updates, Ubuntu:16.10/yakkety-security [amd64]) []\n" - elif cmd.find("--only-upgrade true -s install") > -1: + elif cmd.find("grep -hR security /etc/apt/sources.list") > -1 or cmd.find("grep -hR \"\" /etc/apt/sources.list") > -1: + code = 0 + output = ("deb-src http://azure.archive.ubuntu.com/ubuntu/ jammy-security main restricted\n" + "deb-src http://azure.archive.ubuntu.com/ubuntu/ jammy-security universe\n" + "deb-src http://azure.archive.ubuntu.com/ubuntu/ jammy-security multiverse\n" + "deb-src https://snapshot.ubuntu.com/ubuntu/20240301T000000Z jammy-security universe") + self.write_to_file(os.path.join(self.temp_folder_path, "temp2.list"), output) + elif cmd.find("--only-upgrade true -s install") > -1 or cmd.find("apt-get -y --only-upgrade true upgrade") > -1: code = 0 output = "NOTE: This is only a simulation!\n" + \ " apt-get needs root privileges for real execution.\n" + \ @@ -526,10 +533,6 @@ def run_command_output(self, cmd, no_output=False, chk_err=True): output = " bash | 4.3-14ubuntu1.3 | http://us.archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\n" + \ " bash | 4.3-14ubuntu1.2 | http://security.ubuntu.com/ubuntu xenial-security/main amd64 Packages\n" + \ " bash | 4.3-14ubuntu1 | http://us.archive.ubuntu.com/ubuntu xenial/main amd64 Packages" - elif cmd.find('sudo grep -hR security /etc/apt/sources.list /etc/apt/sources.list.d/ >') > -1: - self.write_to_file(os.path.join(self.temp_folder_path, "temp2.list"), "test temp file 2") - code = 0 - output = "tmp file created" elif cmd.find('sudo apt-get install ubuntu-advantage-tools -y') > -1: code = 0 elif cmd.find('pro security-status --format=json') > -1: diff --git a/src/core/tests/library/RuntimeCompositor.py b/src/core/tests/library/RuntimeCompositor.py index dd27ab33..d8f3e623 100644 --- a/src/core/tests/library/RuntimeCompositor.py +++ b/src/core/tests/library/RuntimeCompositor.py @@ -50,6 +50,11 @@ def __init__(self, argv=Constants.DEFAULT_UNSPECIFIED_VALUE, legacy_mode=False, Constants.SystemPaths.SYSTEMD_ROOT = os.getcwd() # mocking to pass a basic systemd check in Windows self.is_github_runner = os.getenv('RUNNER_TEMP', None) is not None + # speed up test execution + Constants.MAX_FILE_OPERATION_RETRY_COUNT = 1 + Constants.MAX_IMDS_CONNECTION_RETRY_COUNT = 1 + Constants.WAIT_TIME_AFTER_HEALTHSTORE_STATUS_UPDATE_IN_SECS = 0 + if self.is_github_runner: def mkdtemp_runner(): temp_path = os.path.join(os.getenv('RUNNER_TEMP'), str(uuid.uuid4())) @@ -141,6 +146,8 @@ def reconfigure_env_layer_to_legacy_mode(self): self.env_layer.platform = self.legacy_env_layer_extensions.LegacyPlatform() self.env_layer.set_legacy_test_mode() self.env_layer.run_command_output = self.legacy_env_layer_extensions.run_command_output + if os.name == 'nt': + self.env_layer.etc_environment_file_path = os.getcwd() def reconfigure_reboot_manager(self): self.reboot_manager.start_reboot = self.start_reboot diff --git a/src/extension/src/Constants.py b/src/extension/src/Constants.py index a9f57c84..ffbd12a9 100644 --- a/src/extension/src/Constants.py +++ b/src/extension/src/Constants.py @@ -150,6 +150,7 @@ class EnvSettingsFields(EnumBackport): # Public Settings within Config Settings class ConfigPublicSettingsFields(EnumBackport): + cloud_type = "cloudType" operation = "operation" activity_id = "activityId" start_time = "startTime" diff --git a/src/extension/src/ProcessHandler.py b/src/extension/src/ProcessHandler.py index aed742bc..da140603 100644 --- a/src/extension/src/ProcessHandler.py +++ b/src/extension/src/ProcessHandler.py @@ -38,7 +38,8 @@ def get_public_config_settings(config_settings): public_config_settings = {} public_settings_keys = Constants.ConfigPublicSettingsFields if config_settings is not None: - public_config_settings.update({public_settings_keys.operation: config_settings.__getattribute__(public_settings_keys.operation), + public_config_settings.update({public_settings_keys.cloud_type: config_settings.__getattribute__(public_settings_keys.cloud_type), + public_settings_keys.operation: config_settings.__getattribute__(public_settings_keys.operation), public_settings_keys.activity_id: config_settings.__getattribute__(public_settings_keys.activity_id), public_settings_keys.start_time: config_settings.__getattribute__(public_settings_keys.start_time), public_settings_keys.maximum_duration: config_settings.__getattribute__(public_settings_keys.maximum_duration), @@ -48,6 +49,7 @@ def get_public_config_settings(config_settings): public_settings_keys.exclude_patches: config_settings.__getattribute__(public_settings_keys.exclude_patches), public_settings_keys.internal_settings: config_settings.__getattribute__(public_settings_keys.internal_settings), public_settings_keys.maintenance_run_id: config_settings.__getattribute__(public_settings_keys.maintenance_run_id), + public_settings_keys.health_store_id: config_settings.__getattribute__(public_settings_keys.health_store_id), public_settings_keys.patch_mode: config_settings.__getattribute__(public_settings_keys.patch_mode), public_settings_keys.assessment_mode: config_settings.__getattribute__(public_settings_keys.assessment_mode), public_settings_keys.maximum_assessment_interval: config_settings.__getattribute__(public_settings_keys.maximum_assessment_interval)}) diff --git a/src/extension/src/file_handlers/ExtConfigSettingsHandler.py b/src/extension/src/file_handlers/ExtConfigSettingsHandler.py index d858bcb5..ea31bc2a 100644 --- a/src/extension/src/file_handlers/ExtConfigSettingsHandler.py +++ b/src/extension/src/file_handlers/ExtConfigSettingsHandler.py @@ -115,6 +115,7 @@ def read_file(self, seq_no): file_name = str(seq_no) + self.file_ext config_settings_json = self.json_file_handler.get_json_file_content(file_name, self.config_folder, raise_if_not_found=True) if config_settings_json is not None and self.are_config_settings_valid(config_settings_json): + cloud_type = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.cloud_type) operation = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.operation) activity_id = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.activity_id) start_time = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.start_time) @@ -129,12 +130,12 @@ def read_file(self, seq_no): patch_mode = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.patch_mode, raise_if_not_found=False) assessment_mode = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.assessment_mode, raise_if_not_found=False) maximum_assessment_interval = self.get_ext_config_value_safely(config_settings_json, self.public_settings_all_keys.maximum_assessment_interval, raise_if_not_found=False) - config_settings_values = collections.namedtuple("config_settings", [self.public_settings_all_keys.operation, self.public_settings_all_keys.activity_id, self.public_settings_all_keys.start_time, - self.public_settings_all_keys.maximum_duration, self.public_settings_all_keys.reboot_setting, self.public_settings_all_keys.include_classifications, - self.public_settings_all_keys.include_patches, self.public_settings_all_keys.exclude_patches, self.public_settings_all_keys.internal_settings, - self.public_settings_all_keys.maintenance_run_id, self.public_settings_all_keys.health_store_id, self.public_settings_all_keys.patch_mode, - self.public_settings_all_keys.assessment_mode, self.public_settings_all_keys.maximum_assessment_interval]) - return config_settings_values(operation, activity_id, start_time, max_duration, reboot_setting, include_classifications, include_patches, exclude_patches, + config_settings_values = collections.namedtuple("config_settings", [self.public_settings_all_keys.cloud_type, self.public_settings_all_keys.operation, self.public_settings_all_keys.activity_id, + self.public_settings_all_keys.start_time, self.public_settings_all_keys.maximum_duration, self.public_settings_all_keys.reboot_setting, + self.public_settings_all_keys.include_classifications, self.public_settings_all_keys.include_patches, self.public_settings_all_keys.exclude_patches, + self.public_settings_all_keys.internal_settings, self.public_settings_all_keys.maintenance_run_id, self.public_settings_all_keys.health_store_id, + self.public_settings_all_keys.patch_mode, self.public_settings_all_keys.assessment_mode, self.public_settings_all_keys.maximum_assessment_interval]) + return config_settings_values(cloud_type, operation, activity_id, start_time, max_duration, reboot_setting, include_classifications, include_patches, exclude_patches, internal_settings, maintenance_run_id, health_store_id, patch_mode, assessment_mode, maximum_assessment_interval) else: config_invalid_due_to = "no content found in the file" if config_settings_json is None else "settings not in expected format" diff --git a/src/extension/tests/Test_ExtConfigSettingsHandler.py b/src/extension/tests/Test_ExtConfigSettingsHandler.py index 7aef552f..0237e0fa 100644 --- a/src/extension/tests/Test_ExtConfigSettingsHandler.py +++ b/src/extension/tests/Test_ExtConfigSettingsHandler.py @@ -356,7 +356,7 @@ def test_read_all_config_settings_from_file(self): # verify healthStoreId is read successfully self.assertNotEqual(config_settings.__getattribute__(self.config_public_settings_fields.health_store_id), None) - self.assertEqual(config_settings.__getattribute__(self.config_public_settings_fields.health_store_id),"2021-09-15T12:12:14Z") + self.assertEqual(config_settings.__getattribute__(self.config_public_settings_fields.health_store_id),"pub_off_sku_2023.10.11") # verify assessmentMode is read successfully self.assertNotEqual(config_settings.__getattribute__(self.config_public_settings_fields.assessment_mode), None) diff --git a/src/extension/tests/Test_ProcessHandler.py b/src/extension/tests/Test_ProcessHandler.py index f346c2d9..99e19250 100644 --- a/src/extension/tests/Test_ProcessHandler.py +++ b/src/extension/tests/Test_ProcessHandler.py @@ -98,9 +98,21 @@ def test_get_public_config_settings(self): process_handler = ProcessHandler(self.logger, self.env_layer, self.ext_output_status_handler) public_config_settings = process_handler.get_public_config_settings(config_settings) self.assertTrue(public_config_settings is not None) + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.cloud_type), "Azure") self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.operation), "Installation") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.activity_id), "12345-2312-1234-23245-32112") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.start_time), "2021-08-08T12:34:56Z") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.maximum_duration), "PT2H") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.reboot_setting), "IfRequired") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.include_classifications), ["Critical","Security"]) + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.include_patches), ["*ern*=1.2*", "kern*=1.23.45"]) + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.exclude_patches), ["test", "*test"]) + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.internal_settings), "test") self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.maintenance_run_id), "2019-07-20T12:12:14Z") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.health_store_id), "pub_off_sku_2023.10.11") self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.patch_mode), "AutomaticByPlatform") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.assessment_mode), "AutomaticByPlatform") + self.assertEqual(public_config_settings.get(Constants.ConfigPublicSettingsFields.maximum_assessment_interval), "PT3H") def test_get_env_settings(self): # Mock temp folder setup in ExtEnvHandler diff --git a/src/extension/tests/helpers/1234.settings b/src/extension/tests/helpers/1234.settings index 41682dbd..ddc9e0dc 100644 --- a/src/extension/tests/helpers/1234.settings +++ b/src/extension/tests/helpers/1234.settings @@ -1,9 +1,10 @@ { "runtimeSettings": [{ "handlerSettings": { - "protectedSettingsCertThumbprint": "", - "protectedSettings": "", + "protectedSettingsCertThumbprint": null, + "protectedSettings": null, "publicSettings": { + "cloudType": "Azure", "operation": "Installation", "activityId": "12345-2312-1234-23245-32112", "startTime": "2021-08-08T12:34:56Z", @@ -14,7 +15,7 @@ "patchesToExclude": ["test", "*test"], "internalSettings": "test", "maintenanceRunId": "2019-07-20T12:12:14Z", - "healthStoreId": "2021-09-15T12:12:14Z", + "healthStoreId": "pub_off_sku_2023.10.11", "patchMode": "AutomaticByPlatform", "assessmentMode": "AutomaticByPlatform", "maximumAssessmentInterval": "PT3H" From 52b934a1b6150975dccc08faa50d0330e1d0ee40 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Tue, 31 Oct 2023 16:56:15 -0700 Subject: [PATCH 2/2] Linux Patch Extension Release 1.6.49 (#225) --- src/core/src/bootstrap/Constants.py | 2 +- src/extension/src/Constants.py | 2 +- src/extension/src/manifest.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index c84d69fa..67ff5317 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -30,7 +30,7 @@ def __iter__(self): UNKNOWN = "Unknown" # Extension version (todo: move to a different file) - EXT_VERSION = "1.6.48" + EXT_VERSION = "1.6.49" # Runtime environments TEST = 'Test' diff --git a/src/extension/src/Constants.py b/src/extension/src/Constants.py index ffbd12a9..0d6a0e8c 100644 --- a/src/extension/src/Constants.py +++ b/src/extension/src/Constants.py @@ -28,7 +28,7 @@ def __iter__(self): yield item # Extension version (todo: move to a different file) - EXT_VERSION = "1.6.48" + EXT_VERSION = "1.6.49" # Runtime environments TEST = 'Test' diff --git a/src/extension/src/manifest.xml b/src/extension/src/manifest.xml index 49476bc2..2a5202fe 100644 --- a/src/extension/src/manifest.xml +++ b/src/extension/src/manifest.xml @@ -2,7 +2,7 @@ Microsoft.CPlat.Core LinuxPatchExtension - 1.6.48 + 1.6.49 VmRole