Merge branch 'master' into garamrak-obsoletePackages

src/core/src/bootstrap/Constants.py

@@ -30,7 +30,7 @@ def __iter__(self):
     UNKNOWN = "Unknown"
 
     # Extension version (todo: move to a different file)
-    EXT_VERSION = "1.6.47"
+    EXT_VERSION = "1.6.48"
 
     # Runtime environments
     TEST = 'Test'
@@ -51,9 +51,12 @@ def __iter__(self):
     MAX_AUTO_ASSESSMENT_LOGFILE_SIZE_IN_BYTES = 5*1024*1024
     MAX_AUTO_ASSESSMENT_WAIT_FOR_MAIN_CORE_EXEC_IN_MINUTES = 3 * 60
 
-    class Paths(EnumBackport):
+    class SystemPaths(EnumBackport):
         SYSTEMD_ROOT = "/etc/systemd/system/"
 
+    class AzGPSPaths(EnumBackport):
+        EULA_SETTINGS = "/var/lib/azure/linuxpatchextension/patch.eula.settings"
+
     class EnvSettings(EnumBackport):
         LOG_FOLDER = "logFolder"
         CONFIG_FOLDER = "configFolder"
@@ -77,6 +80,11 @@ class ConfigSettings(EnumBackport):
         ASSESSMENT_MODE = 'assessmentMode'
         MAXIMUM_ASSESSMENT_INTERVAL = 'maximumAssessmentInterval'
 
+    class EulaSettings(EnumBackport):
+        ACCEPT_EULA_FOR_ALL_PATCHES = 'AcceptEULAForAllPatches'
+        ACCEPTED_BY = 'AcceptedBy'
+        LAST_MODIFIED = 'LastModified'
+
     TEMP_FOLDER_DIR_NAME = "tmp"
     TEMP_FOLDER_CLEANUP_ARTIFACT_LIST = ["*.list"]
 
@@ -201,6 +209,7 @@ class AutoAssessmentStates(EnumBackport):
     MAX_IMDS_CONNECTION_RETRY_COUNT = 5
     MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5
     MAX_BATCH_SIZE_FOR_PACKAGES = 3
+    MAX_COMPLETE_STATUS_FILES_TO_RETAIN = 10
 
     class PackageClassification(EnumBackport):
         UNCLASSIFIED = 'Unclassified'

src/core/src/core_logic/ExecutionConfig.py

@@ -86,6 +86,9 @@ def __init__(self, env_layer, composite_logger, execution_parameters):
         else:
             self.composite_logger.log_debug("Not executing in auto-assessment mode.")
 
+        # EULA config
+        self.accept_package_eula = self.__is_eula_accepted_for_all_patches()
+
     def __transform_execution_config_for_auto_assessment(self):
         self.activity_id = str(uuid.uuid4())
         self.included_classifications_list = self.included_package_name_mask_list = self.excluded_package_name_mask_list = []
@@ -179,3 +182,31 @@ def __check_and_create_temp_folder_if_not_exists(self):
             self.composite_logger.log_debug("Temp folder does not exist, creating one from extension core. [Path={0}]".format(str(self.temp_folder)))
             os.mkdir(self.temp_folder)
 
+    def __is_eula_accepted_for_all_patches(self):
+        """ Reads customer provided config on EULA acceptance from disk and returns a boolean.
+            NOTE: This is a temporary solution and will be deprecated soon """
+        is_eula_accepted = False
+        try:
+            if os.path.exists(Constants.AzGPSPaths.EULA_SETTINGS):
+                eula_settings = json.loads(self.env_layer.file_system.read_with_retry(Constants.AzGPSPaths.EULA_SETTINGS) or 'null')
+                accept_eula_for_all_patches = self.__fetch_specific_eula_setting(eula_settings, Constants.EulaSettings.ACCEPT_EULA_FOR_ALL_PATCHES)
+                accepted_by = self.__fetch_specific_eula_setting(eula_settings, Constants.EulaSettings.ACCEPTED_BY)
+                last_modified = self.__fetch_specific_eula_setting(eula_settings, Constants.EulaSettings.LAST_MODIFIED)
+                if accept_eula_for_all_patches is not None and accept_eula_for_all_patches in [True, 'True', 'true', '1', 1]:
+                    is_eula_accepted = True
+                self.composite_logger.log_debug("EULA config values from disk: [AcceptEULAForAllPatches={0}] [AcceptedBy={1}] [LastModified={2}]. Computed value of [IsEULAAccepted={3}]"
+                                                .format(str(accept_eula_for_all_patches), str(accepted_by), str(last_modified), str(is_eula_accepted)))
+            else:
+                self.composite_logger.log_debug("No EULA Settings found on the VM. Computed value of [IsEULAAccepted={0}]".format(str(is_eula_accepted)))
+        except Exception as error:
+            self.composite_logger.log_debug("Error occurred while reading and parsing EULA settings. Not accepting EULA for any patch. Error=[{0}]".format(repr(error)))
+
+        return is_eula_accepted
+
+    @staticmethod
+    def __fetch_specific_eula_setting(settings_source, setting_to_fetch):
+        """ Returns the specific setting value from eula_settings_source or None if not found """
+        if settings_source is not None and setting_to_fetch is not None and setting_to_fetch in settings_source:
+            return settings_source[setting_to_fetch]
+        return None
+

src/core/src/core_logic/SystemctlManager.py

@@ -31,7 +31,7 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ
         self.service_desc = service_info.service_desc
         self.service_exec_path = service_info.service_exec_path
 
-        self.__systemd_path = Constants.Paths.SYSTEMD_ROOT
+        self.__systemd_path = Constants.SystemPaths.SYSTEMD_ROOT
         self.systemctl_daemon_reload_cmd = "sudo systemctl daemon-reload"
         self.systemctl_version = "systemctl --version"
 

src/core/src/package_managers/AptitudePackageManager.py

@@ -34,6 +34,10 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ
         super(AptitudePackageManager, self).__init__(env_layer, execution_config, composite_logger, telemetry_writer, status_handler)
 
         security_list_guid = str(uuid.uuid4())
+
+        # Accept EULA (End User License Agreement) as per the EULA settings set by user
+        optional_accept_eula_in_cmd = "ACCEPT_EULA=Y" if execution_config.accept_package_eula else ""
+
         # Repo refresh
         self.repo_refresh = 'sudo apt-get -q update'
 
@@ -44,12 +48,12 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ
         self.single_package_check_versions = 'apt-cache madison <PACKAGE-NAME>'
         self.single_package_find_installed_dpkg = 'sudo dpkg -s <PACKAGE-NAME>'
         self.single_package_find_installed_apt = 'sudo apt list --installed <PACKAGE-NAME>'
-        self.single_package_upgrade_simulation_cmd = '''DEBIAN_FRONTEND=noninteractive apt-get -y --only-upgrade true -s install '''
-        self.single_package_dependency_resolution_template = 'DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF8 apt-get -y --only-upgrade true -s install <PACKAGE-NAME> '
+        self.single_package_upgrade_simulation_cmd = '''DEBIAN_FRONTEND=noninteractive ''' + optional_accept_eula_in_cmd + ''' apt-get -y --only-upgrade true -s install '''
+        self.single_package_dependency_resolution_template = 'DEBIAN_FRONTEND=noninteractive ' + optional_accept_eula_in_cmd + ' LANG=en_US.UTF8 apt-get -y --only-upgrade true -s install <PACKAGE-NAME> '
 
         # Install update
         # --only-upgrade: upgrade only single package (only if it is installed)
-        self.single_package_upgrade_cmd = '''sudo DEBIAN_FRONTEND=noninteractive apt-get -y --only-upgrade true install '''
+        self.single_package_upgrade_cmd = '''sudo DEBIAN_FRONTEND=noninteractive ''' + optional_accept_eula_in_cmd + ''' apt-get -y --only-upgrade true install '''
 
         # Package manager exit code(s)
         self.apt_exitcode_ok = 0
@@ -476,8 +480,9 @@ def get_package_size(self, output):
     def get_current_auto_os_patch_state(self):
         """ Gets the current auto OS update patch state on the machine """
         self.composite_logger.log("Fetching the current automatic OS patch state on the machine...")
-        self.__get_current_auto_os_updates_setting_on_machine()
-        if int(self.unattended_upgrade_value) == 0:
+        if os.path.exists(self.os_patch_configuration_settings_file_path):
+            self.__get_current_auto_os_updates_setting_on_machine()
+        if not os.path.exists(self.os_patch_configuration_settings_file_path) or int(self.unattended_upgrade_value) == 0:
             current_auto_os_patch_state = Constants.AutomaticOSPatchStates.DISABLED
         elif int(self.unattended_upgrade_value) == 1:
             current_auto_os_patch_state = Constants.AutomaticOSPatchStates.ENABLED

src/core/src/service_interfaces/StatusHandler.py

@@ -14,6 +14,7 @@
 #
 # Requires Python 2.7+
 import collections
+import glob
 import json
 import os
 import re
@@ -323,7 +324,7 @@ def set_assessment_substatus_json(self, status=Constants.STATUS_TRANSITIONING, c
         self.__assessment_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_ASSESSMENT_SUMMARY, status, code, json.dumps(self.__assessment_summary_json))
 
         # Update complete status on disk
-        self.__write_complete_status_file()
+        self.__write_status_file()
 
     def __new_assessment_summary_json(self, assessment_packages_json, status, code):
         """ Called by: set_assessment_substatus_json
@@ -372,7 +373,7 @@ def set_installation_substatus_json(self, status=Constants.STATUS_TRANSITIONING,
         self.__installation_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_INSTALLATION_SUMMARY, status, code, json.dumps(self.__installation_summary_json))
 
         # Update complete status on disk
-        self.__write_complete_status_file()
+        self.__write_status_file()
 
     def __new_installation_summary_json(self, installation_packages_json):
         """ Called by: set_installation_substatus_json
@@ -434,7 +435,7 @@ def set_patch_metadata_for_healthstore_substatus_json(self, status=Constants.STA
         self.__metadata_for_healthstore_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_METADATA_FOR_HEALTHSTORE, status, code, json.dumps(self.__metadata_for_healthstore_summary_json))
 
         # Update complete status on disk
-        self.__write_complete_status_file()
+        self.__write_status_file()
 
         # wait period required in cases where we need to ensure HealthStore reads the status from GA
         if wait_after_update:
@@ -467,7 +468,7 @@ def set_configure_patching_substatus_json(self, status=Constants.STATUS_TRANSITI
         self.__configure_patching_substatus_json = self.__new_substatus_json_for_operation(Constants.CONFIGURE_PATCHING_SUMMARY, status, code, json.dumps(self.__configure_patching_summary_json))
 
         # Update complete status on disk
-        self.__write_complete_status_file()
+        self.__write_status_file()
 
     def __new_configure_patching_summary_json(self, automatic_os_patch_state, auto_assessment_state, status, code):
         """ Called by: set_configure_patching_substatus_json
@@ -569,6 +570,9 @@ def load_status_file_components(self, initial_load=False):
 
         self.composite_logger.log_debug("Loading status file components [InitialLoad={0}].".format(str(initial_load)))
 
+        # Remove older complete status files
+        self.__removed_older_complete_status_files(self.execution_config.status_folder)
+
         # Verify the status file exists - if not, reset status file
         if not os.path.exists(self.complete_status_file_path) and initial_load:
             self.composite_logger.log_warning("Status file not found at initial load. Resetting status file to defaults.")
@@ -591,8 +595,7 @@ def load_status_file_components(self, initial_load=False):
                 if self.execution_config.exec_auto_assess_only:
                     self.__installation_substatus_json = complete_status_file_data['status']['substatus'][i]
                 else:
-                    message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                    self.__installation_summary_json = json.loads(message)
+                    self.__installation_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                     self.__installation_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__installation_summary_json['patches'])
                     self.__installation_packages = list(self.__installation_packages_map.values())
                     self.__maintenance_window_exceeded = bool(self.__installation_summary_json['maintenanceWindowExceeded'])
@@ -602,8 +605,7 @@ def load_status_file_components(self, initial_load=False):
                         self.__installation_errors = errors['details']
                         self.__installation_total_error_count = self.__get_total_error_count_from_prev_status(errors['message'])
             if name == Constants.PATCH_ASSESSMENT_SUMMARY:     # if it exists, it must be to spec, or an exception will get thrown
-                message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                self.__assessment_summary_json = json.loads(message)
+                self.__assessment_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                 self.__assessment_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__assessment_summary_json['patches'])
                 self.__assessment_packages = list(self.__assessment_packages_map.values())
                 errors = self.__assessment_summary_json['errors']
@@ -614,19 +616,20 @@ def load_status_file_components(self, initial_load=False):
                 if self.execution_config.exec_auto_assess_only:
                     self.__metadata_for_healthstore_substatus_json = complete_status_file_data['status']['substatus'][i]
                 else:
-                    message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                    self.__metadata_for_healthstore_summary_json = json.loads(message)
+                    self.__metadata_for_healthstore_summary_json = self.__get_substatus_message(complete_status_file_data, i)
             if name == Constants.CONFIGURE_PATCHING_SUMMARY:     # if it exists, it must be to spec, or an exception will get thrown
                 if self.execution_config.exec_auto_assess_only:
                     self.__configure_patching_substatus_json = complete_status_file_data['status']['substatus'][i]
                 else:
-                    message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                    self.__configure_patching_summary_json = json.loads(message)
+                    self.__configure_patching_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                     errors = self.__configure_patching_summary_json['errors']
                     if errors is not None and errors['details'] is not None:
                         self.__configure_patching_errors = errors['details']
                         self.__configure_patching_top_level_error_count = self.__get_total_error_count_from_prev_status(errors['message'])
 
+    def __get_substatus_message(self, status_file_data, index):
+        return json.loads(status_file_data['status']['substatus'][index]['formattedMessage']['message'])
+
     def __load_complete_status_file_data(self, file_path):
         # Read the status file - raise exception on persistent failure
         for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT):
@@ -641,7 +644,7 @@ def __load_complete_status_file_data(self, file_path):
                     raise
         return complete_status_file_data
 
-    def __write_complete_status_file(self):
+    def __write_status_file(self):
         """ Composes and writes the status file from **already up-to-date** in-memory data.
             This is usually the final call to compose and persist after an in-memory data update in a specialized method.
 
@@ -806,3 +809,21 @@ def __set_errors_json(self, error_count_by_operation, errors_by_operation):
         }
     # endregion
 
+    def __removed_older_complete_status_files(self, status_folder):
+        """ Retain 10 latest status complete file and remove other .complete.status files """
+        files_removed = []
+        all_complete_status_files = glob.glob(os.path.join(status_folder, '*.complete.status'))    # Glob return empty list if no file matched pattern
+        if len(all_complete_status_files) <= Constants.MAX_COMPLETE_STATUS_FILES_TO_RETAIN:
+            return
+
+        all_complete_status_files.sort(key=os.path.getmtime, reverse=True)
+        for complete_status_file in all_complete_status_files[Constants.MAX_COMPLETE_STATUS_FILES_TO_RETAIN:]:
+            try:
+                if os.path.exists(complete_status_file):
+                    os.remove(complete_status_file)
+                    files_removed.append(complete_status_file)
+            except Exception as e:
+                self.composite_logger.log_debug("Error deleting complete status file. [File={0} [Exception={1}]]".format(repr(complete_status_file), repr(e)))
+
+        self.composite_logger.log_debug("Cleaned up older complete status files: {0}".format(files_removed))
+