From d2eb5a56b1cbca54526e67f642902bc2a9dfbdef Mon Sep 17 00:00:00 2001 From: Justin Kim Date: Thu, 12 Dec 2024 16:06:15 -0800 Subject: [PATCH] Monitor PkgManager operations Summary: Adding monitoring on failures on * processAll * loadRequiredKmods * unloadKmods * processRpm Some functions have a couple place where it exits or throws on failures. So using SCOPE_* macro. Reviewed By: rationalis Differential Revision: D66711104 fbshipit-source-id: e0e71e7379f3c757d18cac8251e44217877a5ee0 --- .../platform/platform_manager/PkgManager.cpp | 23 +++++++++++++++++ fboss/platform/platform_manager/PkgManager.h | 10 ++++++++ fboss/platform/platform_manager/tests/BUCK | 1 + .../platform_manager/tests/PkgManagerTest.cpp | 25 +++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/fboss/platform/platform_manager/PkgManager.cpp b/fboss/platform/platform_manager/PkgManager.cpp index 89b0e23f3cd43..fb4b6206f50b6 100644 --- a/fboss/platform/platform_manager/PkgManager.cpp +++ b/fboss/platform/platform_manager/PkgManager.cpp @@ -141,6 +141,13 @@ PkgManager::PkgManager( platformFsUtils_(platformFsUtils) {} void PkgManager::processAll() const { + SCOPE_SUCCESS { + fb303::fbData->setCounter(kProcessAllFailure, 0); + }; + SCOPE_FAIL { + fb303::fbData->setCounter(kProcessAllFailure, 1); + }; + if (FLAGS_local_rpm_path.size()) { fb303::fbData->setExportedValue( kBspKmodsRpmName, "local_rpm: " + FLAGS_local_rpm_path); @@ -186,6 +193,13 @@ void PkgManager::processAll() const { } void PkgManager::processRpms() const { + SCOPE_SUCCESS { + fb303::fbData->setCounter(kProcessRpmFailure, 0); + }; + SCOPE_FAIL { + fb303::fbData->setCounter(kProcessRpmFailure, 1); + }; + int exitStatus{0}; if (auto installedRpms = systemInterface_->getInstalledRpms(getKmodsRpmBaseWithKernelName()); @@ -242,6 +256,13 @@ void PkgManager::processLocalRpms() const { } void PkgManager::unloadBspKmods() const { + SCOPE_SUCCESS { + fb303::fbData->setCounter(kUnloadKmodsFailure, 0); + }; + SCOPE_FAIL { + fb303::fbData->setCounter(kUnloadKmodsFailure, 1); + }; + std::string keyword{}; re2::RE2::FullMatch( *platformConfig_.bspKmodsRpmName(), kBspRpmNameRe, &keyword); @@ -305,10 +326,12 @@ void PkgManager::loadRequiredKmods() const { for (const auto& requiredKmod : *platformConfig_.requiredKmodsToLoad()) { XLOG(INFO) << fmt::format("Loading {}", requiredKmod); if (!systemInterface_->loadKmod(requiredKmod)) { + fb303::fbData->setCounter(kLoadKmodsFailure, 1); throw std::runtime_error( fmt::format("Failed to load ({})", requiredKmod)); } } + fb303::fbData->setCounter(kLoadKmodsFailure, 0); } std::string PkgManager::getKmodsRpmName() const { diff --git a/fboss/platform/platform_manager/PkgManager.h b/fboss/platform/platform_manager/PkgManager.h index d611cd8e31d70..6d36c9072c778 100644 --- a/fboss/platform/platform_manager/PkgManager.h +++ b/fboss/platform/platform_manager/PkgManager.h @@ -31,6 +31,16 @@ class SystemInterface { class PkgManager { public: + // ODS Counters + constexpr static auto kProcessAllFailure = + "package_manager.process_all_failure"; + constexpr static auto kLoadKmodsFailure = + "package_manager.load_kmods_failure"; + constexpr static auto kUnloadKmodsFailure = + "package_manager.unload_kmods_failure"; + constexpr static auto kProcessRpmFailure = + "package_manager.process_rpm_failure"; + explicit PkgManager( const PlatformConfig& config, const std::shared_ptr& systemInterface = diff --git a/fboss/platform/platform_manager/tests/BUCK b/fboss/platform/platform_manager/tests/BUCK index 979d29f0b0b00..89f66eab874b5 100644 --- a/fboss/platform/platform_manager/tests/BUCK +++ b/fboss/platform/platform_manager/tests/BUCK @@ -102,6 +102,7 @@ cpp_unittest( ], deps = [ "fbsource//third-party/googletest:gmock", + "//fb303:service_data", "//fboss/platform/platform_manager:pkg_manager", "//thrift/lib/cpp2/protocol:protocol", ], diff --git a/fboss/platform/platform_manager/tests/PkgManagerTest.cpp b/fboss/platform/platform_manager/tests/PkgManagerTest.cpp index 62e008e6c5f38..22d236d884884 100644 --- a/fboss/platform/platform_manager/tests/PkgManagerTest.cpp +++ b/fboss/platform/platform_manager/tests/PkgManagerTest.cpp @@ -1,5 +1,6 @@ // (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. +#include #include #include #include @@ -177,6 +178,8 @@ TEST_F(PkgManagerTest, processRpms) { .WillOnce(Return(0)); EXPECT_CALL(*mockSystemInterface_, depmod()).WillOnce(Return(0)); EXPECT_NO_THROW(pkgManager_.processRpms()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 0); // Installed rpms EXPECT_CALL( *mockSystemInterface_, @@ -199,6 +202,8 @@ TEST_F(PkgManagerTest, processRpms) { .WillOnce(Return(0)); EXPECT_CALL(*mockSystemInterface_, depmod()).WillOnce(Return(0)); EXPECT_NO_THROW(pkgManager_.processRpms()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 0); // Remove installed rpms failed. EXPECT_CALL( *mockSystemInterface_, @@ -213,6 +218,8 @@ TEST_F(PkgManagerTest, processRpms) { "fboss_bsp_kmods-6.4.3-0_fbk1_755_ga25447393a1d-2.4.0-1"})) .WillOnce(Return(1)); EXPECT_THROW(pkgManager_.processRpms(), std::runtime_error); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 1); // depmod failed EXPECT_CALL( *mockSystemInterface_, @@ -234,6 +241,8 @@ TEST_F(PkgManagerTest, processRpms) { .WillOnce(Return(0)); EXPECT_CALL(*mockSystemInterface_, depmod()).WillOnce(Return(1)); EXPECT_NO_THROW(pkgManager_.processRpms()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 0); // Rpm install failed EXPECT_CALL( *mockSystemInterface_, @@ -255,6 +264,8 @@ TEST_F(PkgManagerTest, processRpms) { .Times(3) .WillRepeatedly(Return(1)); EXPECT_THROW(pkgManager_.processRpms(), std::runtime_error); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kProcessRpmFailure), 1); } TEST_F(PkgManagerTest, unloadBspKmods) { @@ -271,6 +282,8 @@ TEST_F(PkgManagerTest, unloadBspKmods) { .WillOnce(Return(std::vector{})); EXPECT_CALL(*mockSystemInterface_, lsmod()).Times(0); EXPECT_NO_THROW(pkgManager_.unloadBspKmods()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 0); // No kmods.json when it should exist EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_)) .WillOnce(Return(std::nullopt)); @@ -282,6 +295,8 @@ TEST_F(PkgManagerTest, unloadBspKmods) { .WillOnce(Return(std::vector{ "fboss_bsp_kmods-6.4.3-0_fbk1_755_ga25447393a1d-2.4.0-1"})); EXPECT_THROW(pkgManager_.unloadBspKmods(), std::runtime_error); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 1); // kmods.json exist and all kmods are loaded EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_)) .WillOnce(Return(jsonBspKmodsFile_)); @@ -294,6 +309,8 @@ TEST_F(PkgManagerTest, unloadBspKmods) { bspKmodsFile_.bspKmods()->size()) .WillRepeatedly(Return(true)); EXPECT_NO_THROW(pkgManager_.unloadBspKmods()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 0); // kmods.json exists but unload fails EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_)) .WillOnce(Return(jsonBspKmodsFile_)); @@ -301,12 +318,16 @@ TEST_F(PkgManagerTest, unloadBspKmods) { .WillOnce(Return(jsonBspKmodsFile_)); EXPECT_CALL(*mockSystemInterface_, unloadKmod(_)).WillOnce(Return(false)); EXPECT_THROW(pkgManager_.unloadBspKmods(), std::runtime_error); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 1); // kmods.json exist and all kmods aren't loaded EXPECT_CALL(*mockPlatformFsUtils_, getStringFileContent(_)) .WillOnce(Return(jsonBspKmodsFile_)); EXPECT_CALL(*mockSystemInterface_, lsmod()).WillOnce(Return("")); EXPECT_CALL(*mockSystemInterface_, unloadKmod(_)).Times(0); EXPECT_NO_THROW(pkgManager_.unloadBspKmods()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kUnloadKmodsFailure), 0); } TEST_F(PkgManagerTest, loadRequiredKmods) { @@ -315,8 +336,12 @@ TEST_F(PkgManagerTest, loadRequiredKmods) { .Times(platformConfig_.requiredKmodsToLoad()->size()) .WillRepeatedly(Return(true)); EXPECT_NO_THROW(pkgManager_.loadRequiredKmods()); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kLoadKmodsFailure), 0); // Load kmods fail EXPECT_CALL(*mockSystemInterface_, loadKmod(_)).WillOnce(Return(false)); EXPECT_THROW(pkgManager_.loadRequiredKmods(), std::runtime_error); + EXPECT_EQ( + facebook::fb303::fbData->getCounter(PkgManager::kLoadKmodsFailure), 1); } }; // namespace facebook::fboss::platform::platform_manager