diff --git a/be/src/storage/lake/meta_file.cpp b/be/src/storage/lake/meta_file.cpp index 542f9e6c13c92..36a2bfc5ca8d5 100644 --- a/be/src/storage/lake/meta_file.cpp +++ b/be/src/storage/lake/meta_file.cpp @@ -25,6 +25,7 @@ #include "util/coding.h" #include "util/defer_op.h" #include "util/raw_container.h" +#include "util/starrocks_metrics.h" #include "util/trace.h" namespace starrocks { @@ -159,6 +160,7 @@ Status MetaFileBuilder::update_num_del_stat(const std::map& se std::string err_msg = fmt::format("unexpected segment id: {} tablet id: {}", each.first, _tablet_meta->id()); LOG(ERROR) << err_msg; + StarRocksMetrics::instance()->primary_key_table_error_state_total.increment(1); if (!config::experimental_lake_ignore_pk_consistency_check) { set_recover_flag(RecoverFlag::RECOVER_WITHOUT_PUBLISH); return Status::InternalError(err_msg); diff --git a/be/src/storage/lake/update_manager.cpp b/be/src/storage/lake/update_manager.cpp index 49332eee7a156..0b71db2b3d434 100644 --- a/be/src/storage/lake/update_manager.cpp +++ b/be/src/storage/lake/update_manager.cpp @@ -84,6 +84,7 @@ StatusOr UpdateManager::prepare_primary_index(const TabletMetadata& _index_cache.update_object_size(index_entry, index.memory_usage()); if (!st.ok()) { if (st.is_already_exist()) { + StarRocksMetrics::instance()->primary_key_table_error_state_total.increment(1); builder->set_recover_flag(RecoverFlag::RECOVER_WITH_PUBLISH); } _index_cache.remove(index_entry); @@ -223,6 +224,7 @@ Status UpdateManager::publish_primary_key_tablet(const TxnLogPB_OpWrite& op_writ "v:$6", tablet->id(), rssid, cur_old, cur_add, cur_new, old_del_vec->version(), metadata.version()); LOG(ERROR) << error_msg; + StarRocksMetrics::instance()->primary_key_table_error_state_total.increment(1); if (!config::experimental_lake_ignore_pk_consistency_check) { builder->set_recover_flag(RecoverFlag::RECOVER_WITH_PUBLISH); return Status::InternalError(error_msg); diff --git a/be/src/storage/tablet_updates.cpp b/be/src/storage/tablet_updates.cpp index c5982121f4124..e2eba8eabce0e 100644 --- a/be/src/storage/tablet_updates.cpp +++ b/be/src/storage/tablet_updates.cpp @@ -3393,6 +3393,7 @@ void TabletUpdates::_print_rowsets(std::vector& rowsets, std::string* } void TabletUpdates::_set_error(const string& msg) { + StarRocksMetrics::instance()->primary_key_table_error_state_total.increment(1); _error_msg = msg; _error = true; _apply_version_changed.notify_all(); diff --git a/be/src/util/starrocks_metrics.cpp b/be/src/util/starrocks_metrics.cpp index 9494ccda01fe7..dab1822c22de4 100644 --- a/be/src/util/starrocks_metrics.cpp +++ b/be/src/util/starrocks_metrics.cpp @@ -96,6 +96,7 @@ StarRocksMetrics::StarRocksMetrics() : _metrics(_s_registry_name) { REGISTER_STARROCKS_METRIC(delta_column_group_get_hit_cache); REGISTER_STARROCKS_METRIC(delta_column_group_get_non_pk_total); REGISTER_STARROCKS_METRIC(delta_column_group_get_non_pk_hit_cache); + REGISTER_STARROCKS_METRIC(primary_key_table_error_state_total); // push request _metrics.register_metric("push_requests_total", MetricLabels().add("status", "SUCCESS"), diff --git a/be/src/util/starrocks_metrics.h b/be/src/util/starrocks_metrics.h index 2a63512e3558f..166dc877f96e0 100644 --- a/be/src/util/starrocks_metrics.h +++ b/be/src/util/starrocks_metrics.h @@ -255,6 +255,7 @@ class StarRocksMetrics { METRIC_DEFINE_INT_COUNTER(delta_column_group_get_hit_cache, MetricUnit::REQUESTS); METRIC_DEFINE_INT_COUNTER(delta_column_group_get_non_pk_total, MetricUnit::REQUESTS); METRIC_DEFINE_INT_COUNTER(delta_column_group_get_non_pk_hit_cache, MetricUnit::REQUESTS); + METRIC_DEFINE_INT_COUNTER(primary_key_table_error_state_total, MetricUnit::REQUESTS); // Gauges METRIC_DEFINE_INT_GAUGE(memory_pool_bytes_total, MetricUnit::BYTES); diff --git a/be/test/storage/lake/meta_file_test.cpp b/be/test/storage/lake/meta_file_test.cpp index 5c544a67a4d82..8cc7d479fee04 100644 --- a/be/test/storage/lake/meta_file_test.cpp +++ b/be/test/storage/lake/meta_file_test.cpp @@ -31,6 +31,7 @@ #include "storage/lake/update_manager.h" #include "testutil/assert.h" #include "testutil/id_generator.h" +#include "util/starrocks_metrics.h" #include "util/uid_util.h" namespace starrocks::lake { @@ -237,4 +238,367 @@ TEST_F(MetaFileTest, test_delvec_read_loop) { } } +<<<<<<< HEAD +======= +TEST_F(MetaFileTest, test_dcg) { + // 1. generate metadata + const int64_t tablet_id = 10001; + auto tablet = std::make_shared(_tablet_manager.get(), tablet_id); + auto metadata = std::make_shared(); + metadata->set_id(tablet_id); + metadata->set_version(10); + metadata->set_next_rowset_id(110); + { + MetaFileBuilder builder(*tablet, metadata); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + + // 2. write first rowset + { + metadata->set_version(11); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("aaa.dat"); + TxnLogPB_OpWrite op_write; + std::map replace_segments; + std::vector orphan_files; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + builder.apply_opwrite(op_write, replace_segments, orphan_files); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + // 3. write dcg + { + metadata->set_version(12); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("bbb.dat"); + TxnLogPB_OpWrite op_write; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + std::vector> filenames; + filenames.emplace_back("aaa.cols", ""); + filenames.emplace_back("bbb.cols", ""); + std::vector> unique_column_id_list; + unique_column_id_list.push_back({3, 4, 5}); + unique_column_id_list.push_back({6, 7, 8}); + builder.append_dcg(110, filenames, unique_column_id_list); + builder.apply_column_mode_partial_update(op_write); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + // <3, 4, 5> -> aaa.cols + // <6, 7, 8> -> bbb.cols + } + { + metadata->set_version(13); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("ccc.dat"); + TxnLogPB_OpWrite op_write; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + std::vector> filenames; + filenames.emplace_back("ccc.cols", ""); + std::vector> unique_column_id_list; + unique_column_id_list.push_back({4, 7}); + builder.append_dcg(110, filenames, unique_column_id_list); + builder.apply_column_mode_partial_update(op_write); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + // <3, 5> -> aaa.cols + // <6, 8> -> bbb.cols + // <4, 7> -> ccc.cols + } + { + metadata->set_version(14); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("ddd.dat"); + TxnLogPB_OpWrite op_write; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + std::vector> filenames; + filenames.emplace_back("ddd.cols", ""); + std::vector> unique_column_id_list; + unique_column_id_list.push_back({3, 5}); + builder.append_dcg(110, filenames, unique_column_id_list); + builder.apply_column_mode_partial_update(op_write); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + auto dcg_ver_iter = metadata->dcg_meta().dcgs().find(110); + EXPECT_TRUE(dcg_ver_iter != metadata->dcg_meta().dcgs().end()); + EXPECT_TRUE(dcg_ver_iter->second.versions_size() == 3); + EXPECT_TRUE(dcg_ver_iter->second.column_files_size() == 3); + EXPECT_TRUE(dcg_ver_iter->second.unique_column_ids_size() == 3); + // <3, 5> -> ddd.cols + // <6, 8> -> bbb.cols + // <4, 7> -> ccc.cols + } + { + auto loader = std::make_unique(metadata); + TabletSegmentId tsid; + tsid.tablet_id = tablet_id; + tsid.segment_id = 110; + DeltaColumnGroupList pdcgs; + EXPECT_TRUE(loader->load(tsid, 1, &pdcgs).ok()); + EXPECT_TRUE(pdcgs.size() == 1); + auto idx = pdcgs[0]->get_column_idx(3); + EXPECT_TRUE("tmp/ddd.cols" == pdcgs[0]->column_files("tmp")[idx.first]); + EXPECT_TRUE("tmp/ddd.cols" == pdcgs[0]->column_file_by_idx("tmp", idx.first).value()); + idx = pdcgs[0]->get_column_idx(4); + EXPECT_TRUE("tmp/ccc.cols" == pdcgs[0]->column_files("tmp")[idx.first]); + EXPECT_TRUE("tmp/ccc.cols" == pdcgs[0]->column_file_by_idx("tmp", idx.first).value()); + idx = pdcgs[0]->get_column_idx(5); + EXPECT_TRUE("tmp/ddd.cols" == pdcgs[0]->column_files("tmp")[idx.first]); + EXPECT_TRUE("tmp/ddd.cols" == pdcgs[0]->column_file_by_idx("tmp", idx.first).value()); + idx = pdcgs[0]->get_column_idx(6); + EXPECT_TRUE("tmp/bbb.cols" == pdcgs[0]->column_files("tmp")[idx.first]); + EXPECT_TRUE("tmp/bbb.cols" == pdcgs[0]->column_file_by_idx("tmp", idx.first).value()); + idx = pdcgs[0]->get_column_idx(7); + EXPECT_TRUE("tmp/ccc.cols" == pdcgs[0]->column_files("tmp")[idx.first]); + EXPECT_TRUE("tmp/ccc.cols" == pdcgs[0]->column_file_by_idx("tmp", idx.first).value()); + idx = pdcgs[0]->get_column_idx(8); + EXPECT_TRUE("tmp/bbb.cols" == pdcgs[0]->column_files("tmp")[idx.first]); + EXPECT_TRUE("tmp/bbb.cols" == pdcgs[0]->column_file_by_idx("tmp", idx.first).value()); + } + // 4. compact (conflict) + { + metadata->set_version(15); + MetaFileBuilder builder(*tablet, metadata); + TxnLogPB_OpCompaction op_compaction; + op_compaction.add_input_rowsets(110); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("eee.dat"); + op_compaction.mutable_output_rowset()->CopyFrom(rowset_metadata); + op_compaction.set_compact_version(13); + EXPECT_TRUE(CompactionUpdateConflictChecker::conflict_check(op_compaction, 111, *metadata, &builder)); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + // 5. compact + { + metadata->set_version(16); + MetaFileBuilder builder(*tablet, metadata); + TxnLogPB_OpCompaction op_compaction; + op_compaction.add_input_rowsets(110); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("fff.dat"); + op_compaction.mutable_output_rowset()->CopyFrom(rowset_metadata); + op_compaction.set_compact_version(14); + EXPECT_FALSE(CompactionUpdateConflictChecker::conflict_check(op_compaction, 111, *metadata, &builder)); + builder.apply_opcompaction(op_compaction, 1, 0); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + { + auto loader = std::make_unique(metadata); + TabletSegmentId tsid; + tsid.tablet_id = tablet_id; + tsid.segment_id = 110; + DeltaColumnGroupList pdcgs; + EXPECT_TRUE(loader->load(tsid, 1, &pdcgs).ok()); + EXPECT_TRUE(pdcgs.empty()); + } + // 6. check orphan files + { + std::set to_check_filenames; + to_check_filenames.insert("aaa.cols"); + to_check_filenames.insert("bbb.cols"); + to_check_filenames.insert("ccc.cols"); + to_check_filenames.insert("ddd.cols"); + to_check_filenames.insert("bbb.dat"); + to_check_filenames.insert("ccc.dat"); + to_check_filenames.insert("ddd.dat"); + to_check_filenames.insert("eee.dat"); + EXPECT_TRUE(metadata->orphan_files_size() == to_check_filenames.size()); + for (const auto& orphan_file : metadata->orphan_files()) { + EXPECT_TRUE(to_check_filenames.count(orphan_file.name()) > 0); + } + } +} + +TEST_F(MetaFileTest, test_unpersistent_del_files_when_compact) { + // 1. generate metadata + const int64_t tablet_id = 10001; + auto tablet = std::make_shared(_tablet_manager.get(), tablet_id); + auto metadata = std::make_shared(); + metadata->set_enable_persistent_index(true); + metadata->set_persistent_index_type(PersistentIndexTypePB::CLOUD_NATIVE); + metadata->set_id(tablet_id); + metadata->set_version(10); + metadata->set_next_rowset_id(110); + { + MetaFileBuilder builder(*tablet, metadata); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + + // 2. write first rowset (110) + { + metadata->set_version(11); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("aaa.dat"); + TxnLogPB_OpWrite op_write; + std::map replace_segments; + std::vector orphan_files; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + builder.apply_opwrite(op_write, replace_segments, orphan_files); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + // 3. write second rowset with del files (111) + { + metadata->set_version(12); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("bbb.dat"); + DelfileWithRowsetId delfile; + delfile.set_name("bbb1.del"); + delfile.set_origin_rowset_id(metadata->next_rowset_id()); + rowset_metadata.add_del_files()->CopyFrom(delfile); + delfile.set_name("bbb2.del"); + rowset_metadata.add_del_files()->CopyFrom(delfile); + TxnLogPB_OpWrite op_write; + std::map replace_segments; + std::vector orphan_files; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + builder.apply_opwrite(op_write, replace_segments, orphan_files); + PersistentIndexSstablePB sstable; + sstable.set_max_rss_rowid((uint64_t)111 << 32); + metadata->mutable_sstable_meta()->add_sstables()->CopyFrom(sstable); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + // 4. compact (112) + { + metadata->set_version(13); + MetaFileBuilder builder(*tablet, metadata); + TxnLogPB_OpCompaction op_compaction; + op_compaction.add_input_rowsets(110); + op_compaction.add_input_rowsets(111); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("ccc.dat"); + op_compaction.mutable_output_rowset()->CopyFrom(rowset_metadata); + op_compaction.set_compact_version(13); + builder.apply_opcompaction(op_compaction, 111, 0); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + // check unpersistent del files + EXPECT_TRUE(metadata->rowsets_size() == 1); + EXPECT_TRUE(metadata->rowsets(0).del_files_size() == 2); + EXPECT_TRUE(metadata->rowsets(0).del_files(0).name() == "bbb1.del"); + EXPECT_TRUE(metadata->rowsets(0).del_files(0).origin_rowset_id() == 111); + EXPECT_TRUE(metadata->rowsets(0).del_files(1).name() == "bbb2.del"); + EXPECT_TRUE(metadata->rowsets(0).del_files(1).origin_rowset_id() == 111); + EXPECT_TRUE(metadata->compaction_inputs_size() == 2); + EXPECT_TRUE(metadata->compaction_inputs(0).del_files_size() == 0); + EXPECT_TRUE(metadata->compaction_inputs(1).del_files_size() == 0); + } + // 5. keep write (113) + { + metadata->set_version(14); + MetaFileBuilder builder(*tablet, metadata); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("ddd.dat"); + TxnLogPB_OpWrite op_write; + std::map replace_segments; + std::vector orphan_files; + op_write.mutable_rowset()->CopyFrom(rowset_metadata); + builder.apply_opwrite(op_write, replace_segments, orphan_files); + PersistentIndexSstablePB sstable; + sstable.set_max_rss_rowid((uint64_t)113 << 32); + metadata->mutable_sstable_meta()->add_sstables()->CopyFrom(sstable); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + } + // 6. compact (114) + { + metadata->set_version(15); + MetaFileBuilder builder(*tablet, metadata); + TxnLogPB_OpCompaction op_compaction; + op_compaction.add_input_rowsets(112); + op_compaction.add_input_rowsets(113); + RowsetMetadataPB rowset_metadata; + rowset_metadata.add_segments("eee.dat"); + op_compaction.mutable_output_rowset()->CopyFrom(rowset_metadata); + op_compaction.set_compact_version(15); + builder.apply_opcompaction(op_compaction, 113, 0); + Status st = builder.finalize(next_id()); + EXPECT_TRUE(st.ok()); + // check unpersistent del files + EXPECT_TRUE(metadata->rowsets_size() == 1); + EXPECT_TRUE(metadata->rowsets(0).del_files_size() == 0); + EXPECT_TRUE(metadata->compaction_inputs(0).del_files_size() == 0); + EXPECT_TRUE(metadata->compaction_inputs(1).del_files_size() == 0); + } +} + +TEST_F(MetaFileTest, test_trim_partial_compaction_last_input_rowset) { + auto metadata = std::make_shared(); + metadata->set_id(9); + metadata->set_version(10); + + TxnLogPB_OpCompaction op_compaction; + op_compaction.add_input_rowsets(1); + op_compaction.add_input_rowsets(11); + op_compaction.add_input_rowsets(22); + op_compaction.mutable_output_rowset()->add_segments("aaa.dat"); + op_compaction.mutable_output_rowset()->add_segments("bbb.dat"); + op_compaction.mutable_output_rowset()->add_segments("ccc.dat"); + op_compaction.mutable_output_rowset()->add_segments("ddd.dat"); + RowsetMetadataPB last_input_rowset_metadata; + + last_input_rowset_metadata.set_id(33); + last_input_rowset_metadata.mutable_segments()->Clear(); + last_input_rowset_metadata.add_segments("aaa.dat"); + last_input_rowset_metadata.add_segments("eee.dat"); + last_input_rowset_metadata.add_segments("fff.dat"); + last_input_rowset_metadata.add_segments("ddd.dat"); + EXPECT_EQ(last_input_rowset_metadata.segments_size(), 4); + // rowset id mismatch + trim_partial_compaction_last_input_rowset(metadata, op_compaction, last_input_rowset_metadata); + EXPECT_EQ(last_input_rowset_metadata.segments_size(), 4); + + last_input_rowset_metadata.set_id(22); + // normal case, duplicate segments will be trimed + trim_partial_compaction_last_input_rowset(metadata, op_compaction, last_input_rowset_metadata); + EXPECT_EQ(last_input_rowset_metadata.segments_size(), 2); + EXPECT_EQ(last_input_rowset_metadata.segments(0), "eee.dat"); + EXPECT_EQ(last_input_rowset_metadata.segments(1), "fff.dat"); + + // no duplicate segments + last_input_rowset_metadata.mutable_segments()->Clear(); + last_input_rowset_metadata.add_segments("xxx.dat"); + last_input_rowset_metadata.add_segments("yyy.dat"); + EXPECT_EQ(last_input_rowset_metadata.segments_size(), 2); + trim_partial_compaction_last_input_rowset(metadata, op_compaction, last_input_rowset_metadata); + EXPECT_EQ(last_input_rowset_metadata.segments_size(), 2); +} + +TEST_F(MetaFileTest, test_error_state) { + // generate metadata + const int64_t tablet_id = 10001; + auto tablet = std::make_shared(_tablet_manager.get(), tablet_id); + auto metadata = std::make_shared(); + metadata->set_id(tablet_id); + metadata->set_version(10); + metadata->set_next_rowset_id(110); + + // add rowset with segment + RowsetMetadataPB rowset_metadata; + rowset_metadata.set_id(110); + rowset_metadata.add_segments("aaa.dat"); + rowset_metadata.add_segments("bbb.dat"); + metadata->add_rowsets()->CopyFrom(rowset_metadata); + std::map segment_id_to_add_dels; + for (int i = 0; i < 10; i++) { + segment_id_to_add_dels[i] = 100; + } + // generate error state + MetaFileBuilder builder(*tablet, metadata); + Status st = builder.update_num_del_stat(segment_id_to_add_dels); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(StarRocksMetrics::instance()->primary_key_table_error_state_total.value() > 0); +} + +>>>>>>> 7c45354ba0 ([Enhancement] add metrics for pk table error state (#52590)) } // namespace starrocks::lake