Skip to content

Commit

Permalink
Merge branch 'satim/version' of https://github.com/Azure/LinuxPatchEx…
Browse files Browse the repository at this point in the history
…tension into satim/version
  • Loading branch information
SathishMSFT committed Oct 7, 2024
2 parents 45b505c + c865bae commit a7fc84f
Show file tree
Hide file tree
Showing 19 changed files with 363 additions and 147 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ jobs:
runs-on: windows-latest
env:
PYTHONTRACEMALLOC: 1
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
steps:
- name: Checkout
uses: actions/checkout@v4
Expand Down Expand Up @@ -46,6 +47,8 @@ jobs:
flags: python39
name: python-39
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Read test output and Check if all tests passed
shell: bash
run: |
Expand All @@ -57,6 +60,8 @@ jobs:
fi
codecov-python-27:
runs-on: windows-latest
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
needs: codecov-python-39
steps:
- name: Checkout
Expand Down Expand Up @@ -112,6 +117,8 @@ jobs:
flags: python27
name: python-27
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Read test output and Check if all tests passed
shell: bash
run: |
Expand Down
5 changes: 4 additions & 1 deletion src/core/src/CoreMain.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def __init__(self, argv):
stdout_file_mirror = bootstrapper.stdout_file_mirror
telemetry_writer = bootstrapper.telemetry_writer
lifecycle_manager = status_handler = execution_config = None
package_manager = None

# Init operation statuses
patch_operation_requested = Constants.UNKNOWN
Expand Down Expand Up @@ -101,7 +102,6 @@ def __init__(self, argv):
patch_installer.mark_installation_completed()
overall_patch_installation_operation_successful = True
self.update_patch_substatus_if_pending(patch_operation_requested, overall_patch_installation_operation_successful, patch_assessment_successful, configure_patching_successful, status_handler, composite_logger)

except Exception as error:
# Privileged operation handling for non-production use
if Constants.EnvLayer.PRIVILEGED_OP_MARKER in repr(error):
Expand Down Expand Up @@ -134,6 +134,9 @@ def __init__(self, argv):
if status_handler is not None:
status_handler.log_truncated_patches()

if package_manager is not None:
package_manager.refresh_repo_safely()

# clean up temp folder of files created by Core after execution completes
if self.is_temp_folder_available(bootstrapper.env_layer, execution_config):
composite_logger.log_debug("Deleting all files of certain format from temp folder [FileFormat={0}][TempFolderLocation={1}]"
Expand Down
31 changes: 15 additions & 16 deletions src/core/src/bootstrap/Constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def __iter__(self):
UNKNOWN = "Unknown"

# Extension version (todo: move to a different file)
EXT_VERSION = "1.6.51"
EXT_VERSION = "1.6.55"

# Runtime environments
TEST = 'Test'
Expand Down Expand Up @@ -219,10 +219,22 @@ class StatusTruncationConfig(EnumBackport):
MAX_INSTALLATION_RETRY_COUNT = 3
MAX_IMDS_CONNECTION_RETRY_COUNT = 5
MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5
MAX_BATCH_SIZE_FOR_PACKAGES = 6
NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL = 3
MAX_COMPLETE_STATUS_FILES_TO_RETAIN = 10

class PackageBatchConfig(EnumBackport):
# Batch Patching Parameters
MAX_BATCH_SIZE_FOR_PACKAGES = 300
MAX_PHASES_FOR_BATCH_PATCHING = 2

# Batch size decay factor is factor to which batch size is decreased in batch patching to install remaining packages in case there
# are some package install failures with original batch size.
BATCH_SIZE_DECAY_FACTOR = 10

# We need to keep some buffer time between calculation of batch size and starting batch patching because after calculating the batch size,
# there would be little time taken before the batch patching is started. The function is_package_install_time_available is called before installing a batch.
# If we do not keep buffer then is_package_install_time_available would return false.
BUFFER_TIME_FOR_BATCH_PATCHING_START_IN_MINUTES = 5

class PackageClassification(EnumBackport):
UNCLASSIFIED = 'Unclassified'
CRITICAL = 'Critical'
Expand Down Expand Up @@ -269,19 +281,6 @@ class PatchAssessmentSummaryStartedBy(EnumBackport):

# Maintenance Window
PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES = 5

# As per telemetry data, when batch size is 3, the average time taken per package installation for different package managers is as follow:
# apt: 43 seconds
# yum: 71 seconds
# zypper: 142 seconds
# Overall, including all package managers, the average time is 51 seconds.
# The average time taken per package installation is average of (total time taken to install the packages) / (total number of packages installed).
# The expected average time should be kept as max of average time taken by different packages managers. The max of average time is taken in zypper i.e. 142 seconds when batch size is 3.
# But as the batch size increases, the time taken to install package will decrease. Also, the average time is taken in consideration in calculating maintenance window cutoff only if the
# batch size is greater than or equal to 4. So, keeping the expected average time as 2 minutes i.e. 120 seconds. It should be fine to keep expected time little lower than actual average time
# observed in telemetry because there is PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES also in the calucalation of maintenance window cut off which will make the overall cutoff time enough to
# install the batch of packages.
PACKAGE_INSTALL_EXPECTED_AVG_TIME_IN_MINUTES = 2

# Package Manager Setting
PACKAGE_MGR_SETTING_REPEAT_PATCH_OPERATION = "RepeatUpdateRun"
Expand Down
31 changes: 8 additions & 23 deletions src/core/src/core_logic/MaintenanceWindow.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,32 +59,16 @@ def get_remaining_time_in_minutes(self, current_time=None, log_to_stdout=False):

return remaining_time_in_minutes

def is_package_install_time_available(self, remaining_time_in_minutes=None, number_of_packages_in_batch=1):
def is_package_install_time_available(self, package_manager, remaining_time_in_minutes=None, number_of_packages_in_batch=1):
"""Check if time still available for package installation"""
cutoff_time_in_minutes = 0

# In the extreme case, all the package installations in the batch might take the maximum time.
# But calculating cutoff time based on max time to install packages for all the packages will make cutoff time very huge
# as it is very unlikely that all the package installations take maximum time.
# Also, as the batch size increases, the expected time taken per package installation decreases due to batch patching.
# So, for batch size less than or equal to 3, calculating cutoff time expecting all packages might take max time to install.
# For larger batch size i.e. 4 or higher, expecting 3 package installations can take max time and rest can take average time to install.
# It is safe assumption that only 3 packages will take max time to install. Even if more number of packages take max time to install then
# due to batch patching in large batch size, the time taken to install a package decreases substantially and hence cutoff time should be enough
# to install the batch of packages.
# PACKAGE_INSTALL_EXPECTED_MAX_TIME is 5 minutes, PACKAGE_INSTALL_EXPECTED_AVG_TIME is 2 minutes
# For different batch size, following would be cutoff:
# Batch Size = 1, Cutoff = 5 * 1 = 5 minutes
# Batch Size = 2, Cutoff = 5 * 2 = 10 minutes
# Batch Size = 3, Cutoff = 5 * 3 = 15 minutes
# Batch Size = 4, Cutoff = (5 * 3) + (2 * 1) = 17 minutes
# Batch Size = 5, Cutoff = (5 * 3) + (2 * 2) = 19 minutes
# Batch Size = 6, Cutoff = (5 * 3) + (2 * 3) = 21 minutes
if number_of_packages_in_batch <= Constants.NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL:
cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES * number_of_packages_in_batch
else:
cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES * Constants.NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL
cutoff_time_in_minutes += Constants.PACKAGE_INSTALL_EXPECTED_AVG_TIME_IN_MINUTES * (number_of_packages_in_batch - Constants.NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL)
# Assuming that one of the packages in the batch could take maximum time to install and rest of the packages take average time.
cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES
if number_of_packages_in_batch > 1:
package_install_expected_avg_time_in_minutes = package_manager.get_package_install_expected_avg_time_in_seconds() / 60
cutoff_time_in_minutes += package_install_expected_avg_time_in_minutes * (number_of_packages_in_batch - 1)

if Constants.REBOOT_SETTINGS[self.execution_config.reboot_setting] != Constants.REBOOT_NEVER:
cutoff_time_in_minutes = cutoff_time_in_minutes + Constants.REBOOT_BUFFER_IN_MINUTES
Expand Down Expand Up @@ -121,4 +105,5 @@ def get_percentage_maintenance_window_used(self):

# Rounding off to one digit after decimal e.g. 14.514372666666667 will become 14.5
percent_maintenance_window_used = round(percent_maintenance_window_used, 1)
return percent_maintenance_window_used
return percent_maintenance_window_used

Loading

0 comments on commit a7fc84f

Please sign in to comment.