From e095658337c91adf833d3e33a60295ccae62a58b Mon Sep 17 00:00:00 2001 From: srlch Date: Mon, 30 Dec 2024 22:02:57 +0800 Subject: [PATCH 1/4] Feature] Support Cluster Snapshot Backup: support system for cluster snapshot backup (part2) Signed-off-by: srlch --- be/src/exec/CMakeLists.txt | 2 + .../schema_cluster_snapshot_jobs_scanner.cpp | 81 ++++++++++++++++++ .../schema_cluster_snapshot_jobs_scanner.h | 37 ++++++++ .../schema_cluster_snapshots_scanner.cpp | 84 +++++++++++++++++++ .../schema_cluster_snapshots_scanner.h | 37 ++++++++ be/src/exec/schema_scanner/schema_helper.cpp | 12 +++ be/src/exec/schema_scanner/schema_helper.h | 7 ++ .../starrocks/catalog/system/SystemId.java | 4 + .../information/ClusterSnapshotJobsTable.java | 43 ++++++++++ .../information/ClusterSnapshotsTable.java | 45 ++++++++++ .../system/information/InfoSchemaDb.java | 2 + .../service/FrontendServiceImpl.java | 14 ++++ gensrc/thrift/Descriptors.thrift | 3 + gensrc/thrift/FrontendService.thrift | 39 +++++++++ .../R/test_cluster_snapshot | 7 ++ .../T/test_cluster_snapshot | 3 + 16 files changed, 420 insertions(+) create mode 100644 be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp create mode 100644 be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h create mode 100644 be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp create mode 100644 be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.h create mode 100644 fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java create mode 100644 test/sql/test_information_schema/R/test_cluster_snapshot create mode 100644 test/sql/test_information_schema/T/test_cluster_snapshot diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 2e3b373adbbf9..24e34d1ecbb3b 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -162,6 +162,8 @@ set(EXEC_FILES schema_scanner/sys_fe_locks.cpp schema_scanner/sys_fe_memory_usage.cpp schema_scanner/schema_temp_tables_scanner.cpp + schema_scanner/schema_cluster_snapshots_scanner.cpp + schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp jdbc_scanner.cpp sorting/compare_column.cpp sorting/merge_column.cpp diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp new file mode 100644 index 0000000000000..4137a970228af --- /dev/null +++ b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp @@ -0,0 +1,81 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h" + +#include "exec/schema_scanner/schema_helper.h" +#include "gen_cpp/FrontendService_types.h" +#include "runtime/runtime_state.h" +#include "runtime/string_value.h" +#include "types/logical_type.h" + +namespace starrocks { + +SchemaScanner::ColumnDesc SchemaClusterSnapshotJobsScanner::_s_columns[] = { + {"snapshot_name", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"job_id", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"created_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"finished_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"state", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"detail_info", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"error_message", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + +}; + +SchemaClusterSnapshotJobsScanner::SchemaClusterSnapshotJobsScanner() + : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc)) {} + +SchemaClusterSnapshotJobsScanner::~SchemaClusterSnapshotJobsScanner() = default; + +Status SchemaClusterSnapshotJobsScanner::start(RuntimeState* state) { + RETURN_IF(!_is_init, Status::InternalError("used before initialized.")); + RETURN_IF(!_param->ip || !_param->port, Status::InternalError("IP or port not exists")); + + RETURN_IF_ERROR(SchemaScanner::start(state)); + RETURN_IF_ERROR(SchemaScanner::init_schema_scanner_state(state)); + + TClusterSnapshotJobsRequest request; + return SchemaHelper::get_cluster_snapshot_jobs_info(_ss_state, request, &_result); +} + +Status SchemaClusterSnapshotJobsScanner::_fill_chunk(ChunkPtr* chunk) { + auto& slot_id_map = (*chunk)->get_slot_id_to_index_map(); + const TClusterSnapshotJobsItem& info = _result.items[_index]; + DatumArray datum_array{ + Slice(info.snapshot_name), info.job_id, + info.created_time, info.finished_time, + Slice(info.state), Slice(info.detail_info), + Slice(info.error_message), + }; + for (const auto& [slot_id, index] : slot_id_map) { + Column* column = (*chunk)->get_column_by_slot_id(slot_id).get(); + column->append_datum(datum_array[slot_id - 1]); + } + _index++; + return {}; +} + +Status SchemaClusterSnapshotJobsScanner::get_next(ChunkPtr* chunk, bool* eos) { + RETURN_IF(!_is_init, Status::InternalError("Used before initialized.")); + RETURN_IF((nullptr == chunk || nullptr == eos), Status::InternalError("input pointer is nullptr.")); + + if (_index >= _result.items.size()) { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_chunk(chunk); +} + +} // namespace starrocks \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h new file mode 100644 index 0000000000000..0b3c97fcae7f7 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h @@ -0,0 +1,37 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "exec/schema_scanner.h" +#include "gen_cpp/FrontendService_types.h" + +namespace starrocks { + +class SchemaClusterSnapshotJobsScanner : public SchemaScanner { +public: + SchemaClusterSnapshotJobsScanner(); + ~SchemaClusterSnapshotJobsScanner() override; + Status start(RuntimeState* state) override; + Status get_next(ChunkPtr* chunk, bool* eos) override; + +private: + Status _fill_chunk(ChunkPtr* chunk); + + size_t _index = 0; + TClusterSnapshotJobsResponse _result; + static SchemaScanner::ColumnDesc _s_columns[]; +}; + +} // namespace starrocks \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp b/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp new file mode 100644 index 0000000000000..ff47671ea4d7c --- /dev/null +++ b/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp @@ -0,0 +1,84 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/schema_scanner/schema_cluster_snapshots_scanner.h" + +#include "exec/schema_scanner/schema_helper.h" +#include "gen_cpp/FrontendService_types.h" +#include "runtime/runtime_state.h" +#include "runtime/string_value.h" +#include "types/logical_type.h" + +namespace starrocks { + +SchemaScanner::ColumnDesc SchemaClusterSnapshotsScanner::_s_columns[] = { + {"snapshot_name", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"snapshot_type", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"created_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"finished_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"fe_jouranl_id", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"starmgr_jouranl_id", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"properties", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"storage_volume", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"storage_path", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + +}; + +SchemaClusterSnapshotsScanner::SchemaClusterSnapshotsScanner() + : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc)) {} + +SchemaClusterSnapshotsScanner::~SchemaClusterSnapshotsScanner() = default; + +Status SchemaClusterSnapshotsScanner::start(RuntimeState* state) { + RETURN_IF(!_is_init, Status::InternalError("used before initialized.")); + RETURN_IF(!_param->ip || !_param->port, Status::InternalError("IP or port not exists")); + + RETURN_IF_ERROR(SchemaScanner::start(state)); + RETURN_IF_ERROR(SchemaScanner::init_schema_scanner_state(state)); + + TClusterSnapshotsRequest request; + return SchemaHelper::get_cluster_snapshots_info(_ss_state, request, &_result); +} + +Status SchemaClusterSnapshotsScanner::_fill_chunk(ChunkPtr* chunk) { + auto& slot_id_map = (*chunk)->get_slot_id_to_index_map(); + const TClusterSnapshotsItem& info = _result.items[_index]; + DatumArray datum_array{ + Slice(info.snapshot_name), Slice(info.snapshot_type), info.created_time, info.finished_time, + + info.fe_jouranl_id, info.starmgr_jouranl_id, Slice(info.properties), Slice(info.storage_volume), + + Slice(info.storage_path), + }; + for (const auto& [slot_id, index] : slot_id_map) { + Column* column = (*chunk)->get_column_by_slot_id(slot_id).get(); + column->append_datum(datum_array[slot_id - 1]); + } + _index++; + return {}; +} + +Status SchemaClusterSnapshotsScanner::get_next(ChunkPtr* chunk, bool* eos) { + RETURN_IF(!_is_init, Status::InternalError("Used before initialized.")); + RETURN_IF((nullptr == chunk || nullptr == eos), Status::InternalError("input pointer is nullptr.")); + + if (_index >= _result.items.size()) { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_chunk(chunk); +} + +} // namespace starrocks \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.h b/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.h new file mode 100644 index 0000000000000..9d2dfcdccec82 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.h @@ -0,0 +1,37 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "exec/schema_scanner.h" +#include "gen_cpp/FrontendService_types.h" + +namespace starrocks { + +class SchemaClusterSnapshotsScanner : public SchemaScanner { +public: + SchemaClusterSnapshotsScanner(); + ~SchemaClusterSnapshotsScanner() override; + Status start(RuntimeState* state) override; + Status get_next(ChunkPtr* chunk, bool* eos) override; + +private: + Status _fill_chunk(ChunkPtr* chunk); + + size_t _index = 0; + TClusterSnapshotsResponse _result; + static SchemaScanner::ColumnDesc _s_columns[]; +}; + +} // namespace starrocks \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index a842fc61eed76..f395f4bf7f131 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -227,6 +227,18 @@ Status SchemaHelper::get_analyze_status(const SchemaScannerState& state, const T }); } +Status SchemaHelper::get_cluster_snapshots_info(const SchemaScannerState& state, const TClusterSnapshotsRequest& req, + TClusterSnapshotsResponse* res) { + return _call_rpc(state, + [&req, &res](FrontendServiceConnection& client) { client->getClusterSnapshotsInfo(*res, req); }); +} + +Status SchemaHelper::get_cluster_snapshot_jobs_info(const SchemaScannerState& state, const TClusterSnapshotJobsRequest& req, + TClusterSnapshotJobsResponse* res) { + return _call_rpc( + state, [&req, &res](FrontendServiceConnection& client) { client->getClusterSnapshotJobsInfo(*res, req); }); +} + void fill_data_column_with_null(Column* data_column) { auto* nullable_column = down_cast(data_column); nullable_column->append_nulls(1); diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 67ad7b0a73a23..5364fbbf2fbc4 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -116,6 +116,13 @@ class SchemaHelper { static Status get_analyze_status(const SchemaScannerState& state, const TAnalyzeStatusReq& var_params, TAnalyzeStatusRes* var_result); + static Status get_cluster_snapshots_info(const SchemaScannerState& state, const TClusterSnapshotsRequest& req, + TClusterSnapshotsResponse* res); + + static Status get_cluster_snapshot_jobs_info(const SchemaScannerState& state, + const TClusterSnapshotJobsRequest& req, + TClusterSnapshotJobsResponse* res); + private: static Status _call_rpc(const SchemaScannerState& state, std::function&)> callback); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/SystemId.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/SystemId.java index 81eda9a50e685..4e36dade60a46 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/SystemId.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/SystemId.java @@ -122,4 +122,8 @@ public class SystemId { // ==================== Statistics =========================== // public static final long COLUMN_STATS_USAGE = 150L; public static final long ANALYZE_STATUS = 151L; + + // ================== Cluster Snapshot ======================= // + public static final long CLUSTER_SNAPSHOTS_ID = 160L; + public static final long CLUSTER_SNAPSHOT_JOBS_ID = 161L; } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java new file mode 100644 index 0000000000000..b1e70513f16d0 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java @@ -0,0 +1,43 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.starrocks.catalog.system.information; + +import com.starrocks.catalog.PrimitiveType; +import com.starrocks.catalog.ScalarType; +import com.starrocks.catalog.Table; +import com.starrocks.catalog.system.SystemId; +import com.starrocks.catalog.system.SystemTable; +import com.starrocks.thrift.TSchemaTableType; + +import static com.starrocks.catalog.system.SystemTable.NAME_CHAR_LEN; +import static com.starrocks.catalog.system.SystemTable.builder; + +public class ClusterSnapshotJobsTable { + public static final String NAME = "cluster_snapshot_jobs"; + + public static SystemTable create() { + return new SystemTable(SystemId.CLUSTER_SNAPSHOT_JOBS_ID, + NAME, + Table.TableType.SCHEMA, + builder() + .column("SNAPSHOT_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("JOB_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CREATE_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("FINISHED_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("STATE", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("DETAIL_INFO", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("ERROR_MESSAGE", ScalarType.createVarchar(NAME_CHAR_LEN)) + .build(), TSchemaTableType.SCH_CLUSTER_SNAPSHOT_JOBS); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java new file mode 100644 index 0000000000000..07463f41eef6f --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java @@ -0,0 +1,45 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.starrocks.catalog.system.information; + +import com.starrocks.catalog.PrimitiveType; +import com.starrocks.catalog.ScalarType; +import com.starrocks.catalog.Table; +import com.starrocks.catalog.system.SystemId; +import com.starrocks.catalog.system.SystemTable; +import com.starrocks.thrift.TSchemaTableType; + +import static com.starrocks.catalog.system.SystemTable.NAME_CHAR_LEN; +import static com.starrocks.catalog.system.SystemTable.builder; + +public class ClusterSnapshotsTable { + public static final String NAME = "cluster_snapshots"; + + public static SystemTable create() { + return new SystemTable(SystemId.CLUSTER_SNAPSHOTS_ID, + NAME, + Table.TableType.SCHEMA, + builder() + .column("SNAPSHOT_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("SNAPSHOT_TYPE", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("CREATE_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("FINISHED_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("FE_JOURNAL_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("STARMGR_JOURNAL_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("PROPERTIES", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("STORAGE_VOLUME", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("STORAGE_PATH", ScalarType.createVarchar(NAME_CHAR_LEN)) + .build(), TSchemaTableType.SCH_CLUSTER_SNAPSHOTS); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/InfoSchemaDb.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/InfoSchemaDb.java index 3ec2b153a9789..22ca207052e60 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/InfoSchemaDb.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/InfoSchemaDb.java @@ -90,6 +90,8 @@ public InfoSchemaDb(String catalogName) { super.registerTableUnlocked(TemporaryTablesTable.create()); super.registerTableUnlocked(ColumnStatsUsageSystemTable.create()); super.registerTableUnlocked(AnalyzeStatusSystemTable.create()); + super.registerTableUnlocked(ClusterSnapshotsTable.create()); + super.registerTableUnlocked(ClusterSnapshotJobsTable.create()); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java index 96e5416f7c98b..daa2a5ae1b45d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java @@ -185,6 +185,10 @@ import com.starrocks.thrift.TBatchReportExecStatusResult; import com.starrocks.thrift.TBeginRemoteTxnRequest; import com.starrocks.thrift.TBeginRemoteTxnResponse; +import com.starrocks.thrift.TClusterSnapshotJobsRequest; +import com.starrocks.thrift.TClusterSnapshotJobsResponse; +import com.starrocks.thrift.TClusterSnapshotsRequest; +import com.starrocks.thrift.TClusterSnapshotsResponse; import com.starrocks.thrift.TColumnDef; import com.starrocks.thrift.TColumnDesc; import com.starrocks.thrift.TColumnStatsUsageReq; @@ -3168,4 +3172,14 @@ static List getPartitionMetaImpl(Collection tabletMe } return result; } + + @Override + public TClusterSnapshotsResponse getClusterSnapshotsInfo(TClusterSnapshotsRequest params) { + return GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().getAllInfo(); + } + + @Override + public TClusterSnapshotJobsResponse getClusterSnapshotJobsInfo(TClusterSnapshotJobsRequest params) { + return GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().getAllJobsInfo(); + } } diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 55b2f4e631545..151cd10339236 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -178,6 +178,9 @@ enum TSchemaTableType { SCH_COLUMN_STATS_USAGE, SCH_ANALYZE_STATUS, + + SCH_CLUSTER_SNAPSHOTS, + SCH_CLUSTER_SNAPSHOT_JOBS, } enum THdfsCompression { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index c1f28a5b4a7fd..0e76d9eaaed28 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1950,6 +1950,42 @@ struct TPartitionMetaResponse { 3: optional list partition_metas; } +struct TClusterSnapshotsItem { + 1: optional string snapshot_name; + 2: optional string snapshot_type; + 3: optional i64 created_time; + 4: optional i64 finished_time; + 5: optional i64 fe_jouranl_id; + 6: optional i64 starmgr_jouranl_id; + 7: optional string properties; + 8: optional string storage_volume; + 9: optional string storage_path; +} + +struct TClusterSnapshotsRequest { +} + +struct TClusterSnapshotsResponse { + 1: optional list items; +} + +struct TClusterSnapshotJobsItem { + 1: optional string snapshot_name; + 2: optional i64 job_id; + 3: optional i64 created_time; + 4: optional i64 finished_time; + 5: optional string state; + 6: optional string detail_info; + 7: optional string error_message; +} + +struct TClusterSnapshotJobsRequest { +} + +struct TClusterSnapshotJobsResponse { + 1: optional list items; +} + service FrontendService { TGetDbsResult getDbNames(1:TGetDbsParams params) TGetTablesResult getTableNames(1:TGetTablesParams params) @@ -2076,5 +2112,8 @@ service FrontendService { TFinishCheckpointResponse finishCheckpoint(1: TFinishCheckpointRequest request) TPartitionMetaResponse getPartitionMeta(TPartitionMetaRequest request) + + TClusterSnapshotsResponse getClusterSnapshotsInfo(1: TClusterSnapshotsRequest request) + TClusterSnapshotJobsResponse getClusterSnapshotJobsInfo(1: TClusterSnapshotJobsRequest request) } diff --git a/test/sql/test_information_schema/R/test_cluster_snapshot b/test/sql/test_information_schema/R/test_cluster_snapshot new file mode 100644 index 0000000000000..2d85c66711271 --- /dev/null +++ b/test/sql/test_information_schema/R/test_cluster_snapshot @@ -0,0 +1,7 @@ +-- name: test_cluster_snapshot +SELECT * FROM INFORMATION_SCHEMA.CLUSTER_SNAPSHOTS; +-- result: +-- !result +SELECT * FROM INFORMATION_SCHEMA.CLUSTER_SNAPSHOT_JOBS; +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_information_schema/T/test_cluster_snapshot b/test/sql/test_information_schema/T/test_cluster_snapshot new file mode 100644 index 0000000000000..8ce33d2e12cc8 --- /dev/null +++ b/test/sql/test_information_schema/T/test_cluster_snapshot @@ -0,0 +1,3 @@ +-- name: test_cluster_snapshot +SELECT * FROM INFORMATION_SCHEMA.CLUSTER_SNAPSHOTS; +SELECT * FROM INFORMATION_SCHEMA.CLUSTER_SNAPSHOT_JOBS; \ No newline at end of file From 7ab83ad66345b59c9f59b851ce010ffb4656121c Mon Sep 17 00:00:00 2001 From: srlch Date: Mon, 30 Dec 2024 22:04:22 +0800 Subject: [PATCH 2/4] fix Signed-off-by: srlch --- .../main/java/com/starrocks/service/FrontendServiceImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java index daa2a5ae1b45d..a4ef1f865a182 100644 --- a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java @@ -3175,11 +3175,10 @@ static List getPartitionMetaImpl(Collection tabletMe @Override public TClusterSnapshotsResponse getClusterSnapshotsInfo(TClusterSnapshotsRequest params) { - return GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().getAllInfo(); + return new TClusterSnapshotsResponse(); } @Override public TClusterSnapshotJobsResponse getClusterSnapshotJobsInfo(TClusterSnapshotJobsRequest params) { - return GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().getAllJobsInfo(); - } + return new TClusterSnapshotJobsResponse(); } From 244d21515e38ed7fbd459c05d508f9662d2e6a3c Mon Sep 17 00:00:00 2001 From: srlch Date: Thu, 2 Jan 2025 12:55:03 +0800 Subject: [PATCH 3/4] fix Signed-off-by: srlch --- be/src/exec/schema_scanner.cpp | 6 ++++++ .../schema_cluster_snapshot_jobs_scanner.cpp | 5 ++--- be/src/exec/schema_scanner/schema_helper.cpp | 7 ++++--- .../java/com/starrocks/service/FrontendServiceImpl.java | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 6ef5f1e85337a..ab500707945b1 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -31,6 +31,8 @@ #include "exec/schema_scanner/schema_be_threads_scanner.h" #include "exec/schema_scanner/schema_be_txns_scanner.h" #include "exec/schema_scanner/schema_charsets_scanner.h" +#include "exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h" +#include "exec/schema_scanner/schema_cluster_snapshots_scanner.h" #include "exec/schema_scanner/schema_collations_scanner.h" #include "exec/schema_scanner/schema_column_stats_usage_scanner.h" #include "exec/schema_scanner/schema_columns_scanner.h" @@ -219,6 +221,10 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return std::make_unique(); case TSchemaTableType::SCH_ANALYZE_STATUS: return std::make_unique(); + case TSchemaTableType::SCH_CLUSTER_SNAPSHOTS: + return std::make_unique(); + case TSchemaTableType::SCH_CLUSTER_SNAPSHOT_JOBS: + return std::make_unique(); default: return std::make_unique(); } diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp index 4137a970228af..2fcb1b4f7265a 100644 --- a/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp @@ -53,9 +53,8 @@ Status SchemaClusterSnapshotJobsScanner::_fill_chunk(ChunkPtr* chunk) { auto& slot_id_map = (*chunk)->get_slot_id_to_index_map(); const TClusterSnapshotJobsItem& info = _result.items[_index]; DatumArray datum_array{ - Slice(info.snapshot_name), info.job_id, - info.created_time, info.finished_time, - Slice(info.state), Slice(info.detail_info), + Slice(info.snapshot_name), info.job_id, info.created_time, + info.finished_time, Slice(info.state), Slice(info.detail_info), Slice(info.error_message), }; for (const auto& [slot_id, index] : slot_id_map) { diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index f395f4bf7f131..9e32c7dfa4d83 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -230,13 +230,14 @@ Status SchemaHelper::get_analyze_status(const SchemaScannerState& state, const T Status SchemaHelper::get_cluster_snapshots_info(const SchemaScannerState& state, const TClusterSnapshotsRequest& req, TClusterSnapshotsResponse* res) { return _call_rpc(state, - [&req, &res](FrontendServiceConnection& client) { client->getClusterSnapshotsInfo(*res, req); }); + [&req, &res](FrontendServiceConnection& client) { client->getClusterSnapshotsInfo(*res, req); }); } -Status SchemaHelper::get_cluster_snapshot_jobs_info(const SchemaScannerState& state, const TClusterSnapshotJobsRequest& req, +Status SchemaHelper::get_cluster_snapshot_jobs_info(const SchemaScannerState& state, + const TClusterSnapshotJobsRequest& req, TClusterSnapshotJobsResponse* res) { return _call_rpc( - state, [&req, &res](FrontendServiceConnection& client) { client->getClusterSnapshotJobsInfo(*res, req); }); + state, [&req, &res](FrontendServiceConnection& client) { client->getClusterSnapshotJobsInfo(*res, req); }); } void fill_data_column_with_null(Column* data_column) { diff --git a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java index a4ef1f865a182..3fec691bc3ddd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java @@ -3181,4 +3181,5 @@ public TClusterSnapshotsResponse getClusterSnapshotsInfo(TClusterSnapshotsReques @Override public TClusterSnapshotJobsResponse getClusterSnapshotJobsInfo(TClusterSnapshotJobsRequest params) { return new TClusterSnapshotJobsResponse(); + } } From db593a27a637d7c5250cf30af26f5b96c363aafb Mon Sep 17 00:00:00 2001 From: srlch Date: Thu, 2 Jan 2025 18:08:36 +0800 Subject: [PATCH 4/4] fix Signed-off-by: srlch --- .../schema_cluster_snapshot_jobs_scanner.cpp | 15 +++++++-------- .../schema_cluster_snapshots_scanner.cpp | 19 +++++++++---------- .../information/ClusterSnapshotJobsTable.java | 2 +- .../information/ClusterSnapshotsTable.java | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp index 2fcb1b4f7265a..fa71c0fbcb2c2 100644 --- a/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp @@ -23,14 +23,13 @@ namespace starrocks { SchemaScanner::ColumnDesc SchemaClusterSnapshotJobsScanner::_s_columns[] = { - {"snapshot_name", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"job_id", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"created_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"finished_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"state", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"detail_info", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"error_message", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - + {"SNAPSHOT_NAME", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"JOB_ID", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"CREATED_TIME", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"FINISHED_TIME", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"STATE", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"DETAIL_INFO", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"ERROR_MESSAGE", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, }; SchemaClusterSnapshotJobsScanner::SchemaClusterSnapshotJobsScanner() diff --git a/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp b/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp index ff47671ea4d7c..4e77498ca7c3d 100644 --- a/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_cluster_snapshots_scanner.cpp @@ -23,16 +23,15 @@ namespace starrocks { SchemaScanner::ColumnDesc SchemaClusterSnapshotsScanner::_s_columns[] = { - {"snapshot_name", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"snapshot_type", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"created_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"finished_time", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"fe_jouranl_id", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"starmgr_jouranl_id", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, - {"properties", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"storage_volume", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - {"storage_path", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, - + {"SNAPSHOT_NAME", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"SNAPSHOT_TYPE", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"CREATED_TIME", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"FINISHED_TIME", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"FE_JOURNAL_ID", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"STARMGR_JOURNAL_ID", TypeDescriptor::from_logical_type(TYPE_BIGINT), sizeof(long), true}, + {"PROPERTIES", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"STORAGE_VOLUME", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, + {"STORAGE_PATH", TypeDescriptor::create_varchar_type(sizeof(StringValue)), sizeof(StringValue), true}, }; SchemaClusterSnapshotsScanner::SchemaClusterSnapshotsScanner() diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java index b1e70513f16d0..3c9f91d2c5f7e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotJobsTable.java @@ -33,7 +33,7 @@ public static SystemTable create() { builder() .column("SNAPSHOT_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) .column("JOB_ID", ScalarType.createType(PrimitiveType.BIGINT)) - .column("CREATE_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CREATED_TIME", ScalarType.createType(PrimitiveType.BIGINT)) .column("FINISHED_TIME", ScalarType.createType(PrimitiveType.BIGINT)) .column("STATE", ScalarType.createVarchar(NAME_CHAR_LEN)) .column("DETAIL_INFO", ScalarType.createVarchar(NAME_CHAR_LEN)) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java index 07463f41eef6f..866035d873f35 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ClusterSnapshotsTable.java @@ -33,7 +33,7 @@ public static SystemTable create() { builder() .column("SNAPSHOT_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) .column("SNAPSHOT_TYPE", ScalarType.createVarchar(NAME_CHAR_LEN)) - .column("CREATE_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CREATED_TIME", ScalarType.createType(PrimitiveType.BIGINT)) .column("FINISHED_TIME", ScalarType.createType(PrimitiveType.BIGINT)) .column("FE_JOURNAL_ID", ScalarType.createType(PrimitiveType.BIGINT)) .column("STARMGR_JOURNAL_ID", ScalarType.createType(PrimitiveType.BIGINT))