From 95f745d3e8c2a76af3a35e00302c5f7e5932286e Mon Sep 17 00:00:00 2001 From: meegoo Date: Fri, 8 Nov 2024 05:11:06 +0800 Subject: [PATCH] update --- be/src/storage/delta_writer.h | 6 +- be/src/storage/lake/async_delta_writer.cpp | 25 +--- .../storage/lake/async_delta_writer_test.cpp | 3 - .../R/test_automatic_partition | 130 ------------------ .../T/test_automatic_partition | 14 -- 5 files changed, 2 insertions(+), 176 deletions(-) diff --git a/be/src/storage/delta_writer.h b/be/src/storage/delta_writer.h index bf0fdfd378544..23cfef96aacdd 100644 --- a/be/src/storage/delta_writer.h +++ b/be/src/storage/delta_writer.h @@ -110,15 +110,11 @@ class DeltaWriter { // [NOT thread-safe] [[nodiscard]] Status commit(); -<<<<<<< HEAD [[nodiscard]] Status flush_memtable_async(bool eos = false); -======= + // Manual flush used by stale memtable flush Status manual_flush(); - Status flush_memtable_async(bool eos = false); ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) - // Rollback all writes and delete the Rowset created by 'commit()', if any. // [thread-safe] // diff --git a/be/src/storage/lake/async_delta_writer.cpp b/be/src/storage/lake/async_delta_writer.cpp index 05f97d9f1a957..84294b5e1bb6e 100644 --- a/be/src/storage/lake/async_delta_writer.cpp +++ b/be/src/storage/lake/async_delta_writer.cpp @@ -124,32 +124,9 @@ inline int AsyncDeltaWriterImpl::execute(void* meta, bthread::TaskIteratortablet_id() << " txn_id: " << delta_writer->txn_id() << ": " << st; } -<<<<<<< HEAD if (st.ok() && iter->flush_after_write) { flush_after_write = true; continue; -======= - case kFlushTask: { - auto flush_task = std::static_pointer_cast(task_ptr); - if (st.ok()) { - st.update(delta_writer->manual_flush()); - } - flush_task->cb(st); - break; - } - case kFinishTask: { - auto finish_task = std::static_pointer_cast(task_ptr); - if (st.ok()) { - auto res = delta_writer->finish_with_txnlog(finish_task->finish_mode); - st.update(res.status()); - LOG_IF(ERROR, !st.ok()) << "Fail to finish write. tablet_id: " << delta_writer->tablet_id() - << " txn_id: " << delta_writer->txn_id() << ": " << st; - finish_task->cb(std::move(res)); - } else { - finish_task->cb(st); - } - break; ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) } if (st.ok() && iter->finish_after_write) { st = delta_writer->finish(); @@ -159,7 +136,7 @@ inline int AsyncDeltaWriterImpl::execute(void* meta, bthread::TaskIteratorcb(st); } if (flush_after_write) { - st = delta_writer->flush_async(); + st = delta_writer->manual_flush(); LOG_IF(ERROR, !st.ok()) << "Fail to flush. tablet_id: " << delta_writer->tablet_id() << " txn_id: " << delta_writer->txn_id() << ": " << st; } diff --git a/be/test/storage/lake/async_delta_writer_test.cpp b/be/test/storage/lake/async_delta_writer_test.cpp index cf8f8d8bbd8a5..6d7f8458f6fb2 100644 --- a/be/test/storage/lake/async_delta_writer_test.cpp +++ b/be/test/storage/lake/async_delta_writer_test.cpp @@ -516,8 +516,6 @@ TEST_F(LakeAsyncDeltaWriterTest, test_concurrent_write_and_close) { SyncPoint::GetInstance()->DisableProcessing(); } -<<<<<<< HEAD -======= TEST_F(LakeAsyncDeltaWriterTest, test_flush) { // Prepare data for writing static const int kChunkSize = 128; @@ -558,5 +556,4 @@ TEST_F(LakeAsyncDeltaWriterTest, test_flush) { delta_writer->close(); } ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) } // namespace starrocks::lake diff --git a/test/sql/test_automatic_bucket/R/test_automatic_partition b/test/sql/test_automatic_bucket/R/test_automatic_partition index e0fd8fb933d36..75de18d68c54d 100644 --- a/test/sql/test_automatic_bucket/R/test_automatic_partition +++ b/test/sql/test_automatic_bucket/R/test_automatic_partition @@ -36,16 +36,6 @@ PROPERTIES ( ); -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_invalid_bucket_size create table t0(k int) properties('bucket_size'='0'); -- result: @@ -68,16 +58,6 @@ alter table t set('bucket_size'='2048'); -- result: -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_automatic_bucket create database kkk; -- result: @@ -110,16 +90,6 @@ alter table t set('bucket_size'='1'); -- result: -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_range_partition @sequential create database ttt; -- result: @@ -179,16 +149,6 @@ select * from t; 2021-01-05 1 -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_list_partition @sequential create database ddd; -- result: @@ -248,16 +208,6 @@ select * from t; 2021-01-03 1 -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_expr_partition @sequential create database eee; -- result: @@ -312,16 +262,6 @@ select * from t; 2021-01-01 1 -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_schema_change @sequential create table t(k date, v int) DUPLICATE KEY(k) PARTITION BY DATE_TRUNC('DAY', `k`) PROPERTIES ( @@ -426,16 +366,6 @@ select * from t; 2021-01-05 None 2 -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_mv @sequential create table t(k date, v int, v1 int) DUPLICATE KEY(k) PARTITION BY DATE_TRUNC('DAY', `k`) PROPERTIES ( @@ -518,16 +448,6 @@ select k, v1 from t; 2021-01-03 2 -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_delete create table t(k int, v int) PROPERTIES ( @@ -566,16 +486,6 @@ select * from t; -- result: -- !result -<<<<<<< HEAD -======= - - - - - - - ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) -- name: test_mutable_bucket @sequential create database ggg; -- result: @@ -661,45 +571,6 @@ alter table t set('mutable_bucket_num'='a'); -- result: E: (5064, 'Getting analyzing error. Detail message: Mutable bucket num: For input string: "a".') -- !result -<<<<<<< HEAD -======= - - - - - --- name: test_tablet_check -create table t(k int)properties('bucket_size'='1'); --- result: --- !result -insert into t values(1); --- result: --- !result -insert into t values(1); --- result: --- !result -insert into t values(1); --- result: --- !result -admin set frontend config('consistency_tablet_meta_check_interval_ms'='100'); --- result: --- !result -select sleep(5); --- result: -1 --- !result -select * from t; --- result: -1 -1 -1 --- !result -admin set frontend config('consistency_tablet_meta_check_interval_ms'='7200000'); --- result: --- !result - - - -- name: test_stale_flush @sequential create table t(k int)properties('bucket_size'='1'); @@ -714,4 +585,3 @@ insert into t select generate_series from TABLE(generate_series(1, 10000000)); update information_schema.be_configs set value=104857600 where name='write_buffer_size'; -- result: -- !result ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613)) diff --git a/test/sql/test_automatic_bucket/T/test_automatic_partition b/test/sql/test_automatic_bucket/T/test_automatic_partition index e6e734849eb77..275aba17c17e7 100644 --- a/test/sql/test_automatic_bucket/T/test_automatic_partition +++ b/test/sql/test_automatic_bucket/T/test_automatic_partition @@ -182,24 +182,10 @@ select count(*) from information_schema.be_tablets t1, information_schema.tables select * from t; alter table t set('mutable_bucket_num'='-1'); -<<<<<<< HEAD alter table t set('mutable_bucket_num'='a'); -======= -alter table t set('mutable_bucket_num'='a'); - --- name: test_tablet_check -create table t(k int)properties('bucket_size'='1'); -insert into t values(1); -insert into t values(1); -insert into t values(1); -admin set frontend config('consistency_tablet_meta_check_interval_ms'='100'); -select sleep(5); -select * from t; -admin set frontend config('consistency_tablet_meta_check_interval_ms'='7200000'); -- name: test_stale_flush @sequential create table t(k int)properties('bucket_size'='1'); update information_schema.be_configs set value=102400 where name='write_buffer_size'; insert into t select generate_series from TABLE(generate_series(1, 10000000)); update information_schema.be_configs set value=104857600 where name='write_buffer_size'; ->>>>>>> 2ec626ec6a ([BugFix] Fix stale mem flush not reduce load memory usage (#52613))