From abf385a018e3b8651cdaae0d6771765b57e7dd64 Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Thu, 7 Dec 2023 07:04:50 -0700 Subject: [PATCH] index btree benchmark --- conanfile.py | 2 +- src/tests/CMakeLists.txt | 5 +- src/tests/btree_helpers/btree_test_helper.hpp | 29 +- src/tests/index_btree_benchmark.cpp | 262 ++++++++++++++++++ src/tests/test_index_btree.cpp | 43 +-- src/tests/test_mem_btree.cpp | 14 +- 6 files changed, 316 insertions(+), 39 deletions(-) create mode 100644 src/tests/index_btree_benchmark.cpp diff --git a/conanfile.py b/conanfile.py index 43ed205db..c50de1b4c 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "4.9.1" + version = "4.9.2" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 42844d22a..decd98b22 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -121,5 +121,8 @@ if (${non_coverage_build}) add_executable(log_store_benchmark) target_sources(log_store_benchmark PRIVATE log_store_benchmark.cpp) target_link_libraries(log_store_benchmark hs_logdev homestore ${COMMON_TEST_DEPS} benchmark::benchmark) - #add_test(NAME LogStoreBench COMMAND test_log_benchmark) + + add_executable(index_btree_benchmark) + target_sources(index_btree_benchmark PRIVATE index_btree_benchmark.cpp) + target_link_libraries(index_btree_benchmark homestore ${COMMON_TEST_DEPS} benchmark::benchmark) endif() diff --git a/src/tests/btree_helpers/btree_test_helper.hpp b/src/tests/btree_helpers/btree_test_helper.hpp index 606aceba0..bc99eca6e 100644 --- a/src/tests/btree_helpers/btree_test_helper.hpp +++ b/src/tests/btree_helpers/btree_test_helper.hpp @@ -15,8 +15,8 @@ *********************************************************************************/ #include #include +#include #include -#include #include #include #include @@ -34,16 +34,16 @@ static constexpr uint32_t g_node_size{4096}; template < typename TestType > -struct BtreeTestHelper : public testing::Test { +struct BtreeHelper { using T = TestType; using K = typename TestType::KeyType; using V = typename TestType::ValueType; using mutex = iomgr::FiberManagerLib::shared_mutex; using op_func_t = std::function< void(void) >; - BtreeTestHelper() : testing::Test(), m_shadow_map{SISL_OPTIONS["num_entries"].as< uint32_t >()} {} + BtreeHelper() : m_shadow_map{SISL_OPTIONS["num_entries"].as< uint32_t >()} {} - void SetUp() override { + void SetUp() { m_cfg.m_leaf_node_type = T::leaf_node_type; m_cfg.m_int_node_type = T::interior_node_type; m_max_range_input = SISL_OPTIONS["num_entries"].as< uint32_t >(); @@ -59,14 +59,14 @@ struct BtreeTestHelper : public testing::Test { }); } - m_operations["put"] = std::bind(&BtreeTestHelper::put_random, this); - m_operations["remove"] = std::bind(&BtreeTestHelper::remove_random, this); - m_operations["range_put"] = std::bind(&BtreeTestHelper::range_put_random, this); - m_operations["range_remove"] = std::bind(&BtreeTestHelper::range_remove_existing_random, this); - m_operations["query"] = std::bind(&BtreeTestHelper::query_random, this); + m_operations["put"] = std::bind(&BtreeHelper::put_random, this); + m_operations["remove"] = std::bind(&BtreeHelper::remove_random, this); + m_operations["range_put"] = std::bind(&BtreeHelper::range_put_random, this); + m_operations["range_remove"] = std::bind(&BtreeHelper::range_remove_existing_random, this); + m_operations["query"] = std::bind(&BtreeHelper::query_random, this); } - void TearDown() override {} + void TearDown() {} protected: std::shared_ptr< typename T::BtreeType > m_bt; @@ -82,6 +82,7 @@ struct BtreeTestHelper : public testing::Test { std::condition_variable m_test_done_cv; std::random_device m_re; + std::atomic< uint32_t > m_num_ops{0}; public: void preload(uint32_t preload_size) { @@ -110,6 +111,8 @@ struct BtreeTestHelper : public testing::Test { LOGINFO("Preload Done"); } + uint32_t get_op_num() const { return m_num_ops.load(); } + ////////////////////// All put operation variants /////////////////////////////// void put(uint64_t k, btree_put_type put_type) { do_put(k, put_type, V::generate_rand()); } @@ -366,6 +369,7 @@ struct BtreeTestHelper : public testing::Test { } } +protected: void run_in_parallel(const std::vector< std::pair< std::string, int > >& op_list) { auto test_count = m_fibers.size(); for (auto it = m_fibers.begin(); it < m_fibers.end(); ++it) { @@ -381,10 +385,13 @@ struct BtreeTestHelper : public testing::Test { // Construct a weighted distribution based on the input frequencies std::discrete_distribution< uint32_t > s_rand_op_generator(weights.begin(), weights.end()); auto m_start_time = Clock::now(); - auto time_to_stop = [this, m_start_time]() {return (get_elapsed_time_sec(m_start_time) > m_run_time);}; + auto time_to_stop = [this, m_start_time]() { + return (get_elapsed_time_sec(m_start_time) > m_run_time); + }; for (uint32_t i = 0; i < num_iters_per_thread && !time_to_stop(); i++) { uint32_t op_idx = s_rand_op_generator(re); (this->m_operations[op_list[op_idx].first])(); + m_num_ops.fetch_add(1); } { std::unique_lock lg(m_test_done_mtx); diff --git a/src/tests/index_btree_benchmark.cpp b/src/tests/index_btree_benchmark.cpp new file mode 100644 index 000000000..4c9ba1d52 --- /dev/null +++ b/src/tests/index_btree_benchmark.cpp @@ -0,0 +1,262 @@ +/********************************************************************************* + * Modifications Copyright 2017-2019 eBay Inc. + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + *********************************************************************************/ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include "test_common/homestore_test_common.hpp" +#include "btree_helpers/btree_test_kvs.hpp" +#include "test_common/range_scheduler.hpp" +#include "btree_helpers/btree_test_helper.hpp" + +using namespace homestore; + +#define CREATE_BENCHMARK_CASE(FUC) \ + template < class BenchmarkType > \ + void FUC(benchmark::State& state) { \ + auto helper = GET_BENCHMARK_HELPER(BenchmarkType); \ + for (auto _ : state) { \ + helper->FUC(); \ + } \ + add_custom_counter< BenchmarkType >(state); \ + } + +#define INDEX_BETREE_BENCHMARK(BTREE_TYPE) \ + BENCHMARK(put< BTREE_TYPE >) \ + ->Setup(BM_Setup< BTREE_TYPE >) \ + ->Teardown(BM_Teardown< BTREE_TYPE >) \ + ->UseRealTime() \ + ->Iterations(1); \ + BENCHMARK(remove< BTREE_TYPE >) \ + ->Setup(BM_SetupAndPreload< BTREE_TYPE >) \ + ->Teardown(BM_Teardown< BTREE_TYPE >) \ + ->UseRealTime() \ + ->Iterations(1); \ + BENCHMARK(range_put< BTREE_TYPE >) \ + ->Setup(BM_SetupAndPreload< BTREE_TYPE >) \ + ->Teardown(BM_Teardown< BTREE_TYPE >) \ + ->UseRealTime() \ + ->Iterations(1); \ + BENCHMARK(query< BTREE_TYPE >) \ + ->Setup(BM_SetupAndPreload< BTREE_TYPE >) \ + ->Teardown(BM_Teardown< BTREE_TYPE >) \ + ->UseRealTime() \ + ->Iterations(1); + +// this is used to splite the setup and teardown from the benchmark to get a more accurate result +void* globle_helper{nullptr}; + +#define GET_BENCHMARK_HELPER(BTREE_TYPE) static_cast< IndexBtreeBenchmark< BTREE_TYPE >* >(globle_helper) + +SISL_LOGGING_INIT(HOMESTORE_LOG_MODS) +std::vector< std::string > test_common::HSTestHelper::s_dev_names; +SISL_OPTIONS_ENABLE(logging, index_btree_benchmark, iomgr, test_common_setup) + +SISL_OPTION_GROUP( + index_btree_benchmark, + (num_iters, "", "num_iters", "number of iterations for rand ops", + ::cxxopts::value< uint32_t >()->default_value("500"), "number"), + (num_entries, "", "num_entries", "number of entries to test with", + ::cxxopts::value< uint32_t >()->default_value("5000"), "number"), + (run_time, "", "run_time", "run time for io", ::cxxopts::value< uint32_t >()->default_value("360000"), "seconds"), + (disable_merge, "", "disable_merge", "disable_merge", ::cxxopts::value< bool >()->default_value("0"), ""), + (operation_list, "", "operation_list", "operation list instead of default created following by percentage", + ::cxxopts::value< std::vector< std::string > >(), "operations [...]"), + (preload_size, "", "preload_size", "number of entries to preload tree with", + ::cxxopts::value< uint32_t >()->default_value("1000"), "number")) + +struct FixedLenBtree { + using BtreeType = IndexTable< TestFixedKey, TestFixedValue >; + using KeyType = TestFixedKey; + using ValueType = TestFixedValue; + static constexpr btree_node_type leaf_node_type = btree_node_type::FIXED; + static constexpr btree_node_type interior_node_type = btree_node_type::FIXED; +}; + +struct VarKeySizeBtree { + using BtreeType = IndexTable< TestVarLenKey, TestFixedValue >; + using KeyType = TestVarLenKey; + using ValueType = TestFixedValue; + static constexpr btree_node_type leaf_node_type = btree_node_type::VAR_KEY; + static constexpr btree_node_type interior_node_type = btree_node_type::VAR_KEY; +}; + +struct VarValueSizeBtree { + using BtreeType = IndexTable< TestVarLenKey, TestVarLenValue >; + using KeyType = TestVarLenKey; + using ValueType = TestVarLenValue; + static constexpr btree_node_type leaf_node_type = btree_node_type::VAR_OBJECT; + static constexpr btree_node_type interior_node_type = btree_node_type::VAR_OBJECT; +}; + +struct VarObjSizeBtree { + using BtreeType = IndexTable< TestVarLenKey, TestVarLenValue >; + using KeyType = TestVarLenKey; + using ValueType = TestVarLenValue; + static constexpr btree_node_type leaf_node_type = btree_node_type::VAR_OBJECT; + static constexpr btree_node_type interior_node_type = btree_node_type::VAR_OBJECT; +}; + +struct PrefixIntervalBtree { + using BtreeType = IndexTable< TestIntervalKey, TestIntervalValue >; + using KeyType = TestIntervalKey; + using ValueType = TestIntervalValue; + static constexpr btree_node_type leaf_node_type = btree_node_type::PREFIX; + static constexpr btree_node_type interior_node_type = btree_node_type::FIXED; +}; + +template < typename TestType > +struct IndexBtreeBenchmark : public BtreeHelper< TestType > { + using T = TestType; + using K = typename TestType::KeyType; + using V = typename TestType::ValueType; + class TestIndexServiceCallbacks : public IndexServiceCallbacks { + public: + TestIndexServiceCallbacks(IndexBtreeBenchmark* test) : m_benchmark(test) {} + std::shared_ptr< IndexTableBase > on_index_table_found(const superblk< index_table_sb >& sb) { + LOGINFO("Index table recovered"); + LOGINFO("Root bnode_id {} version {}", sb->root_node, sb->link_version); + m_benchmark->m_bt = std::make_shared< typename T::BtreeType >(sb, m_benchmark->m_cfg); + return m_benchmark->m_bt; + } + + private: + IndexBtreeBenchmark* m_benchmark; + }; + + IndexBtreeBenchmark() { + this->m_is_multi_threaded = true; + SetUp(); + } + + ~IndexBtreeBenchmark() { TearDown(); } + + void SetUp() { + test_common::HSTestHelper::start_homestore( + "index_btree_benchmark", + {{HS_SERVICE::META, {.size_pct = 10.0}}, + {HS_SERVICE::INDEX, {.size_pct = 70.0, .index_svc_cbs = new TestIndexServiceCallbacks(this)}}}); + + this->m_cfg = BtreeConfig(hs()->index_service().node_size()); + + auto uuid = boost::uuids::random_generator()(); + auto parent_uuid = boost::uuids::random_generator()(); + + BtreeHelper< TestType >::SetUp(); + this->m_bt = std::make_shared< typename T::BtreeType >(uuid, parent_uuid, 0, this->m_cfg); + hs()->index_service().add_index_table(this->m_bt); + } + + void TearDown() { + BtreeHelper< TestType >::TearDown(); + test_common::HSTestHelper::shutdown_homestore(); + } + + void ConcurrentOps(std::vector< std::string > input_ops) { + std::vector< std::pair< std::string, int > > ops; + if (SISL_OPTIONS.count("operation_list")) { + input_ops = SISL_OPTIONS["operation_list"].as< std::vector< std::string > >(); + } + int total = std::accumulate(input_ops.begin(), input_ops.end(), 0, [](int sum, const auto& str) { + std::vector< std::string > tokens; + boost::split(tokens, str, boost::is_any_of(":")); + if (tokens.size() == 2) { + try { + return sum + std::stoi(tokens[1]); + } catch (const std::exception&) { + // Invalid frequency, ignore this element + } + } + return sum; // Ignore malformed strings + }); + + std::transform(input_ops.begin(), input_ops.end(), std::back_inserter(ops), [total](const auto& str) { + std::vector< std::string > tokens; + boost::split(tokens, str, boost::is_any_of(":")); + if (tokens.size() == 2) { + try { + return std::make_pair(tokens[0], (int)(100.0 * std::stoi(tokens[1]) / total)); + } catch (const std::exception&) { + // Invalid frequency, ignore this element + } + } + return std::make_pair(std::string(), 0); + }); + + this->run_in_parallel(ops); + } + + void put() { this->ConcurrentOps({"put:100"}); } + + void remove() { this->ConcurrentOps({"remove:100"}); } + + void range_put() { this->ConcurrentOps({"range_put:100"}); } + + void query() { this->ConcurrentOps({"query:100"}); } +}; + +template < class BenchmarkType > +void BM_Setup(const benchmark::State& state) { + globle_helper = new IndexBtreeBenchmark< BenchmarkType >(); +} + +template < class BenchmarkType > +void BM_SetupAndPreload(const benchmark::State& state) { + BM_Setup< BenchmarkType >(state); + auto helper = GET_BENCHMARK_HELPER(BenchmarkType); + helper->preload(SISL_OPTIONS["preload_size"].as< uint32_t >()); +} + +template < class BenchmarkType > +void BM_Teardown(const benchmark::State& state) { + delete GET_BENCHMARK_HELPER(BenchmarkType); +} + +template < class BenchmarkType > +void add_custom_counter(benchmark::State& state) { + auto helper = GET_BENCHMARK_HELPER(BenchmarkType); + auto totol_ops = helper->get_op_num(); + state.counters["thread_num"] = SISL_OPTIONS["num_threads"].as< uint32_t >(); + state.counters["fiber_num"] = SISL_OPTIONS["num_fibers"].as< uint32_t >(); + state.counters["total_ops"] = totol_ops; + state.counters["rate"] = benchmark::Counter(totol_ops, benchmark::Counter::kIsRate); + state.counters["InvRate"] = + benchmark::Counter(totol_ops, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); +} + +CREATE_BENCHMARK_CASE(put) +CREATE_BENCHMARK_CASE(remove) +CREATE_BENCHMARK_CASE(range_put) +CREATE_BENCHMARK_CASE(query) + +INDEX_BETREE_BENCHMARK(FixedLenBtree) +INDEX_BETREE_BENCHMARK(VarKeySizeBtree) +INDEX_BETREE_BENCHMARK(VarValueSizeBtree) +INDEX_BETREE_BENCHMARK(VarObjSizeBtree) +INDEX_BETREE_BENCHMARK(PrefixIntervalBtree) + +int main(int argc, char** argv) { + SISL_OPTIONS_LOAD(argc, argv, logging, index_btree_benchmark, iomgr, test_common_setup); + // sisl::logging::SetLogger("index_btree_benchmark"); + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); +} diff --git a/src/tests/test_index_btree.cpp b/src/tests/test_index_btree.cpp index e110a931d..80487ebe9 100644 --- a/src/tests/test_index_btree.cpp +++ b/src/tests/test_index_btree.cpp @@ -46,19 +46,20 @@ std::vector< std::string > test_common::HSTestHelper::s_dev_names; // TODO Add tests to do write,remove after recovery. // TODO Test with var len key with io mgr page size is 512. -SISL_OPTION_GROUP(test_index_btree, - (num_iters, "", "num_iters", "number of iterations for rand ops", - ::cxxopts::value< uint32_t >()->default_value("500"), "number"), - (num_entries, "", "num_entries", "number of entries to test with", - ::cxxopts::value< uint32_t >()->default_value("5000"), "number"), - (run_time, "", "run_time", "run time for io", ::cxxopts::value< uint32_t >()->default_value("360000"), "seconds"), - (disable_merge, "", "disable_merge", "disable_merge", ::cxxopts::value< bool >()->default_value("0"), ""), - (operation_list, "", "operation_list", "operation list instead of default created following by percentage", - ::cxxopts::value< std::vector< std::string > >(), "operations [...]"), - (preload_size, "", "preload_size", "number of entries to preload tree with", - ::cxxopts::value< uint32_t >()->default_value("1000"), "number"), - (seed, "", "seed", "random engine seed, use random if not defined", - ::cxxopts::value< uint64_t >()->default_value("0"), "number")) +SISL_OPTION_GROUP( + test_index_btree, + (num_iters, "", "num_iters", "number of iterations for rand ops", + ::cxxopts::value< uint32_t >()->default_value("500"), "number"), + (num_entries, "", "num_entries", "number of entries to test with", + ::cxxopts::value< uint32_t >()->default_value("5000"), "number"), + (run_time, "", "run_time", "run time for io", ::cxxopts::value< uint32_t >()->default_value("360000"), "seconds"), + (disable_merge, "", "disable_merge", "disable_merge", ::cxxopts::value< bool >()->default_value("0"), ""), + (operation_list, "", "operation_list", "operation list instead of default created following by percentage", + ::cxxopts::value< std::vector< std::string > >(), "operations [...]"), + (preload_size, "", "preload_size", "number of entries to preload tree with", + ::cxxopts::value< uint32_t >()->default_value("1000"), "number"), + (seed, "", "seed", "random engine seed, use random if not defined", + ::cxxopts::value< uint64_t >()->default_value("0"), "number")) struct FixedLenBtreeTest { using BtreeType = IndexTable< TestFixedKey, TestFixedValue >; @@ -101,7 +102,7 @@ struct PrefixIntervalBtreeTest { }; template < typename TestType > -struct BtreeTest : public BtreeTestHelper< TestType > { +struct BtreeTest : public BtreeHelper< TestType >, public ::testing::Test { using T = TestType; using K = typename TestType::KeyType; using V = typename TestType::ValueType; @@ -120,6 +121,8 @@ struct BtreeTest : public BtreeTestHelper< TestType > { BtreeTest* m_test; }; + BtreeTest() : testing::Test() {} + void SetUp() override { test_common::HSTestHelper::start_homestore( "test_index_btree", @@ -140,14 +143,14 @@ struct BtreeTest : public BtreeTestHelper< TestType > { homestore::hs()->resource_mgr().reset_dirty_buf_qd(); // Create index table and attach to index service. - BtreeTestHelper< TestType >::SetUp(); + BtreeHelper< TestType >::SetUp(); this->m_bt = std::make_shared< typename T::BtreeType >(uuid, parent_uuid, 0, this->m_cfg); hs()->index_service().add_index_table(this->m_bt); LOGINFO("Added index table to index service"); } void TearDown() override { - BtreeTestHelper< TestType >::TearDown(); + BtreeHelper< TestType >::TearDown(); test_common::HSTestHelper::shutdown_homestore(); } @@ -456,7 +459,7 @@ TYPED_TEST(BtreeTest, ThreadedCpFlush) { } template < typename TestType > -struct BtreeConcurrentTest : public BtreeTestHelper< TestType > { +struct BtreeConcurrentTest : public BtreeHelper< TestType >, public ::testing::Test { using T = TestType; using K = typename TestType::KeyType; @@ -475,7 +478,7 @@ struct BtreeConcurrentTest : public BtreeTestHelper< TestType > { BtreeConcurrentTest* m_test; }; - BtreeConcurrentTest() { this->m_is_multi_threaded = true; } + BtreeConcurrentTest() : testing::Test() { this->m_is_multi_threaded = true; } void SetUp() override { test_common::HSTestHelper::start_homestore( @@ -497,14 +500,14 @@ struct BtreeConcurrentTest : public BtreeTestHelper< TestType > { homestore::hs()->resource_mgr().reset_dirty_buf_qd(); // Create index table and attach to index service. - BtreeTestHelper< TestType >::SetUp(); + BtreeHelper< TestType >::SetUp(); this->m_bt = std::make_shared< typename T::BtreeType >(uuid, parent_uuid, 0, this->m_cfg); hs()->index_service().add_index_table(this->m_bt); LOGINFO("Added index table to index service"); } void TearDown() override { - BtreeTestHelper< TestType >::TearDown(); + BtreeHelper< TestType >::TearDown(); test_common::HSTestHelper::shutdown_homestore(); } }; diff --git a/src/tests/test_mem_btree.cpp b/src/tests/test_mem_btree.cpp index 94e78d53c..89e9ef40b 100644 --- a/src/tests/test_mem_btree.cpp +++ b/src/tests/test_mem_btree.cpp @@ -92,13 +92,15 @@ struct PrefixIntervalBtreeTest { }; template < typename TestType > -struct BtreeTest : public BtreeTestHelper< TestType > { +struct BtreeTest : public BtreeHelper< TestType >, public ::testing::Test { using T = TestType; using K = typename TestType::KeyType; using V = typename TestType::ValueType; + BtreeTest() : testing::Test() {} + void SetUp() override { - BtreeTestHelper< TestType >::SetUp(); + BtreeHelper< TestType >::SetUp(); this->m_bt = std::make_shared< typename T::BtreeType >(this->m_cfg); this->m_bt->init(nullptr); } @@ -282,12 +284,12 @@ TYPED_TEST(BtreeTest, RandomRemoveRange) { } template < typename TestType > -struct BtreeConcurrentTest : public BtreeTestHelper< TestType > { +struct BtreeConcurrentTest : public BtreeHelper< TestType >, public ::testing::Test { using T = TestType; using K = typename TestType::KeyType; using V = typename TestType::ValueType; - BtreeConcurrentTest() { this->m_is_multi_threaded = true; } + BtreeConcurrentTest() : testing::Test() { this->m_is_multi_threaded = true; } void SetUp() override { LOGINFO("Starting iomgr with {} threads", SISL_OPTIONS["num_threads"].as< uint32_t >()); @@ -297,13 +299,13 @@ struct BtreeConcurrentTest : public BtreeTestHelper< TestType > { .app_mem_size_mb = 0, .hugepage_size_mb = 0}); - BtreeTestHelper< TestType >::SetUp(); + BtreeHelper< TestType >::SetUp(); this->m_bt = std::make_shared< typename T::BtreeType >(this->m_cfg); this->m_bt->init(nullptr); } void TearDown() override { - BtreeTestHelper< TestType >::TearDown(); + BtreeHelper< TestType >::TearDown(); iomanager.stop(); } };