From 2730aae7223198776eabf9dbf1a32a9dbd2f9176 Mon Sep 17 00:00:00 2001 From: feng-j678 <127874208+feng-j678@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:42:59 -0700 Subject: [PATCH 1/4] address the stringio getvalue not found error (#275) --- src/core/tests/Test_ConfigurePatchingProcessor.py | 9 +++++++-- src/core/tests/Test_MaintenanceWindow.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/core/tests/Test_ConfigurePatchingProcessor.py b/src/core/tests/Test_ConfigurePatchingProcessor.py index 76db40e3..11f1d38c 100644 --- a/src/core/tests/Test_ConfigurePatchingProcessor.py +++ b/src/core/tests/Test_ConfigurePatchingProcessor.py @@ -18,7 +18,11 @@ import re import unittest import sys -from io import StringIO +# Conditional import for StringIO +try: + from StringIO import StringIO # Python 2 +except ImportError: + from io import StringIO # Python 3 from core.src.CoreMain import CoreMain from core.src.bootstrap.Constants import Constants @@ -317,6 +321,7 @@ def test_configure_patching_with_patch_mode_and_assessment_mode_by_platform(self def test_configure_patching_raise_exception_auto_os_patch_state(self): # arrange capture std IO captured_output = StringIO() + original_stdout = sys.stdout sys.stdout = captured_output argument_composer = ArgumentComposer() @@ -334,7 +339,7 @@ def test_configure_patching_raise_exception_auto_os_patch_state(self): runtime.configure_patching_processor.start_configure_patching() # restore sdt.out ouptput - sys.stdout = sys.__stdout__ + sys.stdout = original_stdout # assert output = captured_output.getvalue() diff --git a/src/core/tests/Test_MaintenanceWindow.py b/src/core/tests/Test_MaintenanceWindow.py index e9e0fb29..18fe7857 100644 --- a/src/core/tests/Test_MaintenanceWindow.py +++ b/src/core/tests/Test_MaintenanceWindow.py @@ -15,12 +15,17 @@ # Requires Python 2.7+ import datetime -from io import StringIO import sys +# Conditional import for StringIO +try: + from StringIO import StringIO # Python 2 +except ImportError: + from io import StringIO # Python 3 import unittest from core.tests.library.ArgumentComposer import ArgumentComposer from core.tests.library.RuntimeCompositor import RuntimeCompositor + class TestMaintenanceWindow(unittest.TestCase): def setUp(self): pass @@ -67,6 +72,7 @@ def test_RemainingTime_after_duration_complete(self): def test_RemainingTime_log_to_stdout_true(self): # Arrange, Capture stdout captured_output = StringIO() + original_output = sys.stdout sys.stdout = captured_output # Redirect stdout to the StringIO object argument_composer = ArgumentComposer() @@ -78,7 +84,7 @@ def test_RemainingTime_log_to_stdout_true(self): remaining_time = runtime.maintenance_window.get_remaining_time_in_minutes(current_time, log_to_stdout=True) # Restore stdout - sys.stdout = sys.__stdout__ + sys.stdout = original_output # Assert output = captured_output.getvalue() From 96f6359806f5b37f5d2624d06790391b153d3ab1 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Tue, 5 Nov 2024 13:53:19 -0800 Subject: [PATCH 2/4] Matching 20% increase in throttling limit at VM Agent (#277) --- src/core/src/bootstrap/Constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index c016c027..99d58939 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -315,8 +315,8 @@ class PatchOperationErrorCodes(EnumBackport): TELEMETRY_DIR_SIZE_LIMIT_IN_CHARS = 41943040 TELEMETRY_BUFFER_FOR_DROPPED_COUNT_MSG_IN_CHARS = 25 # buffer for the chars dropped text added at the end of the truncated telemetry message TELEMETRY_EVENT_COUNTER_MSG_SIZE_LIMIT_IN_CHARS = 15 # buffer for telemetry event counter text added at the end of every message sent to telemetry - TELEMETRY_MAX_EVENT_COUNT_THROTTLE = 60 - TELEMETRY_MAX_TIME_IN_SECONDS_FOR_EVENT_COUNT_THROTTLE = 60 + TELEMETRY_MAX_EVENT_COUNT_THROTTLE = 360 + TELEMETRY_MAX_TIME_IN_SECONDS_FOR_EVENT_COUNT_THROTTLE = 300 # Telemetry Event Level class TelemetryEventLevel(EnumBackport): From c49a676315d5347cd3ead2e9880adeb7a8bbdf79 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Tue, 5 Nov 2024 14:11:03 -0800 Subject: [PATCH 3/4] Remove external dependencies from coverage, increase default target (#245) * Remove external dependencies from coverage * Retry path specification * Update ci.yml Reviewing what happens with dependencies * Update codecov.yml Experimenting * Including contributing.md * Trying exclusion per docs * Trying this another way --- .github/codecov.yml | 7 +++++-- CONTRIBUTING.md | 50 +++++++++++++++++++++++++++++++++++---------- codecov.yml | 3 +++ 3 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml index 9c4549fb..3c17c5f3 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -4,6 +4,9 @@ comment: require_changes: false github_checks: annotations: false +ignore: + - "**/distro.py" + - "src/external_dependencies" coverage: precision: 2 round: down @@ -11,7 +14,7 @@ coverage: status: project: default: - target: 75% + target: 90% threshold: 0% if_ci_failed: error if_not_found: failure @@ -26,4 +29,4 @@ coverage: target: 100% threshold: 0% if_ci_failed: error - if_not_found: failure \ No newline at end of file + if_not_found: failure diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b73616e..617ed650 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ If you would like to become an active contributor to this project, please follow [Making Changes](#making-changes) - [Pull Requests](#pull-requests) - [Pull Request Guidelines](#pull-request-guidelines) + - [PR Planning / AzGPS Engineering Requirements](#pr-planning--azgps-engineering-requirements) - [Cleaning up commits](#cleaning-up-commits) - [General guidelines](#general-guidelines) - [Testing guidelines](#testing-guidelines) @@ -41,6 +42,12 @@ To open your own pull request, click [here](https://github.com/Azure/LinuxPatchE ### Pull Request Guidelines +#### PR Planning / AzGPS Engineering Requirements + +The backlog for the Linux Patch Extension is tracked internally at Microsoft (by the Azure Guest Patching Service team in the Azure Core Compute Platform). There are also closed-source dependencies with the Compute Platform on this extension, and there are internal quality-control requirements set on end-to-end scenarios. + +To ensure smooth engineering, if there's a change required on the extension, please proactively start a conversation with the engineering team via the Issues page. We cannot provide an SLA on unsolicited PRs if not discussed prior, so please reach out as early as possible. + #### Cleaning up Commits If you are thinking about making a large change, **break up the change into small, logical, testable chunks, and organize your pull requests accordingly**. @@ -51,28 +58,49 @@ If you find yourself creating a pull request and are unable to see all the chang If splitting up the pull request is not an option, we recommend **creating individual commits for different parts of the pull request, which can be reviewed individually on GitHub**. -For more information on cleaning up the commits in a pull request, such as how to rebase, squash, and cherry-pick, click [here](https://github.com/Azure/azure-powershell/blob/dev/documentation/development-docs/cleaning-up-commits.md). +For more information on cleaning up the commits in a pull request, such as how to rebase, squash, and cherry-pick, click [here](https://github.com/Azure/azure-powershell/blob/main/documentation/development-docs/cleaning-up-commits.md). #### General guidelines The following guidelines must be followed in **EVERY** pull request that is opened. -- Title of the pull request is clear and informative -- There are a small number of commits that each have an informative message -- A description of the changes the pull request makes is included, and a reference to the issue being resolved, if the change address any -- All files have the Microsoft copyright header +- Title of the pull request is clear and informative. +- There are a small number of commits that each have an informative message. +- A description of the changes the pull request makes is included, and a reference to the issue being resolved, if the change address any. +- All files have the Microsoft copyright header. #### Testing Guidelines The following guidelines must be followed in **EVERY** pull request that is opened. - Pull request includes test coverage for the included changes -- All existing tests must continue to pass successfully on both Python 2.7+ and Python 3.x. -- Code must have been tested on all supported distributions and versions of those distributions that have not reached end of life. The distribution test matrix is as follows: + - All new code introduced **must not** reduce the measured code coverage of any file or of the master branch as a whole. + - Code coverage threshold to be met: **95% on all new code.** Efforts are ongoing to meet or exceed this target on existing code. +- All existing tests must continue to pass successfully on both Python 2.7+ and Python 3.x (latest version). +- Code must have been tested on all supported distributions and versions of those distributions that have not reached end of life. The primary distribution test matrix is as follows: Dist | Version | -----|---------| -Ubuntu Server | 16.04-LTS, 18.04-LTS, 20.04-LTS -Red Hat Enterprise Linux | 6 (x86/x64), 7 (x64), 8 (x64) -CentOS | 6 (x86/x64), 7 (x64), 8 (x64) -SUSE Linux Enterprise Server | 11 (x86/x64), 12 (x64), 15 (x64) +Ubuntu Server | 20.04-LTS, 22.04-LTS, 24.04-LTS +Red Hat Enterprise Linux | 8 (x64), 9 (x64) +SUSE Linux Enterprise Server | 15 (x64) + +The following distribution-versions are supported under **extended support policies** from the vendor and **must also be tested**: + +Dist | Version | Comment | +-----|---------|---------| +Ubuntu Server | 16.04-LTS, 18.04-LTS | Until Apr 2nd, 2026 & Apr 1st, 2028 (resp.) +Red Hat Enterprise Linux | 7 (x64) | Until Jun 30th, 2024 +CentOS | 7 (x64) | Until Jun 30th, 2024 +SUSE Linux Enterprise Server | 12 (x64) | Until Oct 31st, 2027 + +The following distributions have been **EXCLUDED** from support due to end of life and end of extended support: + +Dist | Version | Comment | +-----|---------|---------| +Red Hat Enterprise Linux | 6 (x86/x64) | Ended Nov 30th, 2020 +CentOS | 6 (x86/x64), 8 (x64) | Ended Nov 30th, 2020 & Dec 31st, 2021 (resp.) +SUSE Linux Enterprise Server | 11 (x86/x64) | Ended Mar 31st, 2022 + +**All dates listed are accurate as of March 12th, 2024. Please refer to official distribution vendor guidance for up-to-date information.** + diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..82777754 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,3 @@ +ignore: + - "**/distro.py" + - "src/external_dependencies" \ No newline at end of file From e89747a52d35379e11dc3dc0bd68de95e8ca3bcc Mon Sep 17 00:00:00 2001 From: feng-j678 <127874208+feng-j678@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:59:44 -0800 Subject: [PATCH 4/4] Pr1/increase_RebootManager_and_PatchAssessor_code coverage (#267) * add ut for add_error_to_status * add test_start_reboot() * add logic to rset sys.stdout and add dummy test in rebootmanager.py * modify start_reboot for testing * use import six, StringIO works in py2 py3 * add ut for start_reboot raise exception * create a new method to preserve start_reboot * restore the sys.__stdout * revert import to use io instead of six * add ut for rebootifrequired * test stringIO import * restore configurepatchingprocessor and maintenancewindow tests as master * refactor the test assessor throws exception --- src/core/src/core_logic/RebootManager.py | 2 -- src/core/tests/Test_PatchAssessor.py | 12 +++++++-- src/core/tests/Test_RebootManager.py | 30 ++++++++++++++++++++- src/core/tests/library/RuntimeCompositor.py | 8 ++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/core/src/core_logic/RebootManager.py b/src/core/src/core_logic/RebootManager.py index fc41de60..292e1a14 100644 --- a/src/core/src/core_logic/RebootManager.py +++ b/src/core/src/core_logic/RebootManager.py @@ -15,8 +15,6 @@ # Requires Python 2.7+ """Reboot management""" -import datetime -import subprocess import time from core.src.bootstrap.Constants import Constants diff --git a/src/core/tests/Test_PatchAssessor.py b/src/core/tests/Test_PatchAssessor.py index 40f1c22d..6d0a9f11 100644 --- a/src/core/tests/Test_PatchAssessor.py +++ b/src/core/tests/Test_PatchAssessor.py @@ -23,7 +23,7 @@ from core.src.service_interfaces.TelemetryWriter import TelemetryWriter from core.tests.library.ArgumentComposer import ArgumentComposer from core.tests.library.RuntimeCompositor import RuntimeCompositor -from core.src.core_logic.Stopwatch import Stopwatch + class TestPatchAssessor(unittest.TestCase): def setUp(self): @@ -165,7 +165,6 @@ def test_write_assessment_perf_logs(self): err_msg = "{0}=".format(str(Constants.PerfLogTrackerParams.ERROR_MSG)) self.assertTrue(err_msg in str(self.runtime.patch_assessor.stopwatch.task_details)) - def test_stopwatch_properties_assessment_fail(self): self.runtime.set_legacy_test_type('UnalignedPath') self.assertRaises(Exception, self.runtime.patch_assessor.start_assessment) @@ -181,6 +180,15 @@ def test_raise_if_min_python_version_not_met(self): self.runtime.patch_assessor.start_assessment() self.assertEqual(str(context.exception), Constants.PYTHON_NOT_COMPATIBLE_ERROR_MSG.format(sys.version_info)) + def test_patch_assessment_throws_exception(self): + self.runtime.package_manager.get_all_updates = lambda: self.raise_ex() + + with self.assertRaises(Exception) as context: + self.runtime.patch_assessor.start_assessment() + + self.assertIn(Constants.ERROR_ADDED_TO_STATUS, repr(context.exception)) + self.assertEqual(context.exception.args[1], "[{0}]".format(Constants.ERROR_ADDED_TO_STATUS)) + def raise_ex(self): raise Exception() diff --git a/src/core/tests/Test_RebootManager.py b/src/core/tests/Test_RebootManager.py index f3174f3c..a692fb45 100644 --- a/src/core/tests/Test_RebootManager.py +++ b/src/core/tests/Test_RebootManager.py @@ -13,8 +13,8 @@ # limitations under the License. # # Requires Python 2.7+ - import unittest + from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer from core.tests.library.RuntimeCompositor import RuntimeCompositor @@ -113,6 +113,34 @@ def test_reboot_always_time_not_available(self): self.assertEqual(reboot_manager.start_reboot_if_required_and_time_available(10), False) runtime.stop() + def test_reboot_if_required_no_reboot_pending(self): + reboot_setting_in_api = 'IfRequired' + argument_composer = ArgumentComposer() + argument_composer.reboot_setting = reboot_setting_in_api + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + reboot_manager = runtime.reboot_manager + + # Validate single reboot scenario + runtime.status_handler.is_reboot_pending = False + self.assertEqual(reboot_manager.start_reboot_if_required_and_time_available(20), False) + runtime.stop() + + def test_start_reboot_raise_exception(self): + reboot_setting_in_api = 'Always' + argument_composer = ArgumentComposer() + argument_composer.reboot_setting = reboot_setting_in_api + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + Constants.REBOOT_WAIT_TIMEOUT_IN_MINUTES = -20 + + with self.assertRaises(Exception) as context: + runtime.use_original_rm_start_reboot() + runtime.reboot_manager.start_reboot() + + # assert + self.assertIn("Reboot failed to proceed on the machine in a timely manner.", repr(context.exception)) + self.assertEqual(context.exception.args[1], "[{0}]".format(Constants.ERROR_ADDED_TO_STATUS)) + runtime.stop() + if __name__ == '__main__': unittest.main() diff --git a/src/core/tests/library/RuntimeCompositor.py b/src/core/tests/library/RuntimeCompositor.py index d8f3e623..86dae841 100644 --- a/src/core/tests/library/RuntimeCompositor.py +++ b/src/core/tests/library/RuntimeCompositor.py @@ -43,6 +43,7 @@ class RuntimeCompositor(object): def __init__(self, argv=Constants.DEFAULT_UNSPECIFIED_VALUE, legacy_mode=False, package_manager_name=Constants.APT, vm_cloud_type=Constants.VMCloudType.AZURE): # Init data + self.original_rm_start_reboot = None self.current_env = Constants.DEV os.environ[Constants.LPE_ENV_VARIABLE] = self.current_env self.argv = argv if argv != Constants.DEFAULT_UNSPECIFIED_VALUE else ArgumentComposer().get_composed_arguments() @@ -150,11 +151,18 @@ def reconfigure_env_layer_to_legacy_mode(self): self.env_layer.etc_environment_file_path = os.getcwd() def reconfigure_reboot_manager(self): + # Preserve the original reboot manager start_reboot method + self.original_rm_start_reboot = self.reboot_manager.start_reboot + + # Reassign start_reboot to a new mock method self.reboot_manager.start_reboot = self.start_reboot def start_reboot(self, message="Test initiated reboot mock"): self.status_handler.set_installation_reboot_status(Constants.RebootStatus.STARTED) + def use_original_rm_start_reboot(self): + self.reboot_manager.start_reboot = self.original_rm_start_reboot + def reconfigure_package_manager(self): self.backup_get_current_auto_os_patch_state = self.package_manager.get_current_auto_os_patch_state self.package_manager.get_current_auto_os_patch_state = self.get_current_auto_os_patch_state