Merge branch 'master' into kjohn-excludeexternal

.github/workflows/ci.yml

@@ -9,6 +9,7 @@ jobs:
     runs-on: windows-latest
     env:
       PYTHONTRACEMALLOC: 1
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -46,6 +47,8 @@ jobs:
           flags: python39
           name: python-39
           fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       - name: Read test output and Check if all tests passed
         shell: bash
         run: |
@@ -57,6 +60,9 @@ jobs:
           fi
   codecov-python-27:
     runs-on: windows-latest
+    env:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    needs: codecov-python-39
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -111,6 +117,8 @@ jobs:
           flags: python27
           name: python-27
           fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       - name: Read test output and Check if all tests passed
         shell: bash
         run: |

src/core/src/CoreMain.py

@@ -29,6 +29,7 @@ def __init__(self, argv):
         stdout_file_mirror = bootstrapper.stdout_file_mirror
         telemetry_writer = bootstrapper.telemetry_writer
         lifecycle_manager = status_handler = execution_config = None
+        package_manager = None
 
         # Init operation statuses
         patch_operation_requested = Constants.UNKNOWN
@@ -101,7 +102,6 @@ def __init__(self, argv):
                     patch_installer.mark_installation_completed()
                     overall_patch_installation_operation_successful = True
                 self.update_patch_substatus_if_pending(patch_operation_requested, overall_patch_installation_operation_successful, patch_assessment_successful, configure_patching_successful, status_handler, composite_logger)
-
         except Exception as error:
             # Privileged operation handling for non-production use
             if Constants.EnvLayer.PRIVILEGED_OP_MARKER in repr(error):
@@ -134,6 +134,9 @@ def __init__(self, argv):
             if status_handler is not None:
                 status_handler.log_truncated_patches()
 
+            if package_manager is not None:
+                package_manager.refresh_repo_safely()
+
             # clean up temp folder of files created by Core after execution completes
             if self.is_temp_folder_available(bootstrapper.env_layer, execution_config):
                 composite_logger.log_debug("Deleting all files of certain format from temp folder [FileFormat={0}][TempFolderLocation={1}]"

src/core/src/bootstrap/Constants.py

@@ -31,7 +31,7 @@ def __iter__(self):
     UNKNOWN = "Unknown"
 
     # Extension version (todo: move to a different file)
-    EXT_VERSION = "1.6.53"
+    EXT_VERSION = "1.6.55"
 
     # Runtime environments
     TEST = 'Test'
@@ -219,10 +219,22 @@ class StatusTruncationConfig(EnumBackport):
     MAX_INSTALLATION_RETRY_COUNT = 3
     MAX_IMDS_CONNECTION_RETRY_COUNT = 5
     MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5
-    MAX_BATCH_SIZE_FOR_PACKAGES = 6
-    NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL = 3
     MAX_COMPLETE_STATUS_FILES_TO_RETAIN = 10
 
+    class PackageBatchConfig(EnumBackport):
+        # Batch Patching Parameters
+        MAX_BATCH_SIZE_FOR_PACKAGES = 300
+        MAX_PHASES_FOR_BATCH_PATCHING = 2
+
+        # Batch size decay factor is factor to which batch size is decreased in batch patching to install remaining packages in case there
+        # are some package install failures with original batch size.
+        BATCH_SIZE_DECAY_FACTOR = 10
+
+        # We need to keep some buffer time between calculation of batch size and starting batch patching because after calculating the batch size,
+        # there would be little time taken before the batch patching is started. The function is_package_install_time_available is called before installing a batch.
+        # If we do not keep buffer then is_package_install_time_available would return false.
+        BUFFER_TIME_FOR_BATCH_PATCHING_START_IN_MINUTES = 5
+
     class PackageClassification(EnumBackport):
         UNCLASSIFIED = 'Unclassified'
         CRITICAL = 'Critical'
@@ -269,19 +281,6 @@ class PatchAssessmentSummaryStartedBy(EnumBackport):
 
     # Maintenance Window
     PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES = 5
-
-    # As per telemetry data, when batch size is 3, the average time taken per package installation for different package managers is as follow:
-    # apt: 43 seconds
-    # yum: 71 seconds
-    # zypper: 142 seconds
-    # Overall, including all package managers, the average time is 51 seconds.
-    # The average time taken per package installation is average of (total time taken to install the packages) / (total number of packages installed).
-    # The expected average time should be kept as max of average time taken by different packages managers. The max of average time is taken in zypper i.e. 142 seconds when batch size is 3.
-    # But as the batch size increases, the time taken to install package will decrease. Also, the average time is taken in consideration in calculating maintenance window cutoff only if the 
-    # batch size is greater than or equal to 4. So, keeping the expected average time as 2 minutes i.e. 120 seconds. It should be fine to keep expected time little lower than actual average time 
-    # observed in telemetry because there is PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES also in the calucalation of maintenance window cut off which will make the overall cutoff time enough to 
-    # install the batch of packages.
-    PACKAGE_INSTALL_EXPECTED_AVG_TIME_IN_MINUTES = 2
 
     # Package Manager Setting
     PACKAGE_MGR_SETTING_REPEAT_PATCH_OPERATION = "RepeatUpdateRun"

src/core/src/core_logic/MaintenanceWindow.py

@@ -59,32 +59,16 @@ def get_remaining_time_in_minutes(self, current_time=None, log_to_stdout=False):
 
         return remaining_time_in_minutes
 
-    def is_package_install_time_available(self, remaining_time_in_minutes=None, number_of_packages_in_batch=1):
+    def is_package_install_time_available(self, package_manager, remaining_time_in_minutes=None, number_of_packages_in_batch=1):
         """Check if time still available for package installation"""
-        cutoff_time_in_minutes = 0
-
         # In the extreme case, all the package installations in the batch might take the maximum time. 
         # But calculating cutoff time based on max time to install packages for all the packages will make cutoff time very huge 
         # as it is very unlikely that all the package installations take maximum time.
-        # Also, as the batch size increases, the expected time taken per package installation decreases due to batch patching.
-        # So, for batch size less than or equal to 3, calculating cutoff time expecting all packages might take max time to install.
-        # For larger batch size i.e. 4 or higher, expecting 3 package installations can take max time and rest can take average time to install.
-        # It is safe assumption that only 3 packages will take max time to install. Even if more number of packages take max time to install then
-        # due to batch patching in large batch size, the time taken to install a package decreases substantially and hence cutoff time should be enough
-        # to install the batch of packages.
-        # PACKAGE_INSTALL_EXPECTED_MAX_TIME is 5 minutes, PACKAGE_INSTALL_EXPECTED_AVG_TIME is 2 minutes
-        # For different batch size, following would be cutoff:
-        # Batch Size = 1, Cutoff = 5 * 1 = 5 minutes
-        # Batch Size = 2, Cutoff = 5 * 2 = 10 minutes
-        # Batch Size = 3, Cutoff = 5 * 3 = 15 minutes
-        # Batch Size = 4, Cutoff = (5 * 3) + (2 * 1) = 17 minutes
-        # Batch Size = 5, Cutoff = (5 * 3) + (2 * 2) = 19 minutes
-        # Batch Size = 6, Cutoff = (5 * 3) + (2 * 3) = 21 minutes
-        if number_of_packages_in_batch <= Constants.NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL:
-            cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES * number_of_packages_in_batch
-        else:
-            cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES * Constants.NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL
-            cutoff_time_in_minutes += Constants.PACKAGE_INSTALL_EXPECTED_AVG_TIME_IN_MINUTES * (number_of_packages_in_batch - Constants.NUMBER_OF_PACKAGES_IN_BATCH_COULD_TAKE_MAX_TIME_TO_INSTALL)
+        # Assuming that one of the packages in the batch could take maximum time to install and rest of the packages take average time.
+        cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES
+        if number_of_packages_in_batch > 1:
+            package_install_expected_avg_time_in_minutes = package_manager.get_package_install_expected_avg_time_in_seconds() / 60
+            cutoff_time_in_minutes += package_install_expected_avg_time_in_minutes * (number_of_packages_in_batch - 1)
 
         if Constants.REBOOT_SETTINGS[self.execution_config.reboot_setting] != Constants.REBOOT_NEVER:
             cutoff_time_in_minutes = cutoff_time_in_minutes + Constants.REBOOT_BUFFER_IN_MINUTES
@@ -121,4 +105,5 @@ def get_percentage_maintenance_window_used(self):
 
         # Rounding off to one digit after decimal e.g. 14.514372666666667 will become 14.5
         percent_maintenance_window_used = round(percent_maintenance_window_used, 1)
-        return percent_maintenance_window_used
+        return percent_maintenance_window_used
+