From 7271bc367991633658806980211828c3872065d5 Mon Sep 17 00:00:00 2001 From: srlch <111035020+srlch@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:02:28 +0800 Subject: [PATCH 1/2] [Feature] Support Cluster Snapshot Backup: SQL Interface and meta data (part 1) (#54447) Signed-off-by: srlch (cherry picked from commit be70af0d8c5dbecaad922780734231372a5d83da) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java --- .../com/starrocks/journal/JournalEntity.java | 5 + .../lake/snapshot/ClusterSnapshotMgr.java | 118 +++++++++ .../starrocks/persist/ClusterSnapshotLog.java | 78 ++++++ .../java/com/starrocks/persist/EditLog.java | 9 + .../com/starrocks/persist/OperationType.java | 3 + .../persist/metablock/SRMetaBlockID.java | 2 + .../com/starrocks/qe/DDLStmtExecutor.java | 32 +++ .../com/starrocks/server/GlobalStateMgr.java | 11 + .../com/starrocks/sql/analyzer/Analyzer.java | 15 ++ .../sql/analyzer/ClusterSnapshotAnalyzer.java | 62 +++++ .../ast/AdminSetAutomatedSnapshotOffStmt.java | 33 +++ .../ast/AdminSetAutomatedSnapshotOnStmt.java | 40 ++++ .../com/starrocks/sql/ast/AstVisitor.java | 8 + .../com/starrocks/sql/parser/AstBuilder.java | 19 ++ .../com/starrocks/sql/parser/StarRocks.g4 | 14 +- .../com/starrocks/sql/parser/StarRocksLex.g4 | 2 + .../starrocks/lake/ClusterSnapshotTest.java | 225 ++++++++++++++++++ .../starrocks/persist/OperationTypeTest.java | 1 + 18 files changed, 675 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/persist/ClusterSnapshotLog.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ClusterSnapshotAnalyzer.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOffStmt.java create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOnStmt.java create mode 100644 fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java diff --git a/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java b/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java index 168af2944afcb..5a75285881c82 100644 --- a/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java @@ -76,6 +76,7 @@ import com.starrocks.persist.CancelDecommissionDiskInfo; import com.starrocks.persist.CancelDisableDiskInfo; import com.starrocks.persist.ChangeMaterializedViewRefreshSchemeLog; +import com.starrocks.persist.ClusterSnapshotLog; import com.starrocks.persist.ColocatePersistInfo; import com.starrocks.persist.ColumnRenameInfo; import com.starrocks.persist.ConsistencyCheckInfo; @@ -775,6 +776,10 @@ public void readFields(DataInput in) throws IOException { data = DropWarehouseLog.read(in); break; } + case OperationType.OP_CLUSTER_SNAPSHOT_LOG: { + data = ClusterSnapshotLog.read(in); + break; + } default: { if (Config.metadata_ignore_unknown_operation_type) { LOG.warn("UNKNOWN Operation Type {}", opCode); diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java new file mode 100644 index 0000000000000..058ac4ff43a3e --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java @@ -0,0 +1,118 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.lake.snapshot; + +import com.google.gson.annotations.SerializedName; +import com.starrocks.persist.ClusterSnapshotLog; +import com.starrocks.persist.ImageWriter; +import com.starrocks.persist.gson.GsonPostProcessable; +import com.starrocks.persist.metablock.SRMetaBlockEOFException; +import com.starrocks.persist.metablock.SRMetaBlockException; +import com.starrocks.persist.metablock.SRMetaBlockID; +import com.starrocks.persist.metablock.SRMetaBlockReader; +import com.starrocks.persist.metablock.SRMetaBlockWriter; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOffStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOnStmt; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; + +// only used for AUTOMATED snapshot for now +public class ClusterSnapshotMgr implements GsonPostProcessable { + public static final Logger LOG = LogManager.getLogger(ClusterSnapshotMgr.class); + public static final String AUTOMATED_NAME_PREFIX = "automated_cluster_snapshot"; + + @SerializedName(value = "automatedSnapshotSvName") + private String automatedSnapshotSvName = ""; + + public ClusterSnapshotMgr() {} + + // Turn on automated snapshot, use stmt for extension in future + public void setAutomatedSnapshotOn(AdminSetAutomatedSnapshotOnStmt stmt) { + String storageVolumeName = stmt.getStorageVolumeName(); + setAutomatedSnapshotOn(storageVolumeName); + + ClusterSnapshotLog log = new ClusterSnapshotLog(); + log.setCreateSnapshotNamePrefix(AUTOMATED_NAME_PREFIX, storageVolumeName); + GlobalStateMgr.getCurrentState().getEditLog().logClusterSnapshotLog(log); + } + + protected void setAutomatedSnapshotOn(String storageVolumeName) { + automatedSnapshotSvName = storageVolumeName; + } + + public String getAutomatedSnapshotSvName() { + return automatedSnapshotSvName; + } + + public boolean isAutomatedSnapshotOn() { + return automatedSnapshotSvName != null && !automatedSnapshotSvName.isEmpty(); + } + + // Turn off automated snapshot, use stmt for extension in future + public void setAutomatedSnapshotOff(AdminSetAutomatedSnapshotOffStmt stmt) { + setAutomatedSnapshotOff(); + + ClusterSnapshotLog log = new ClusterSnapshotLog(); + log.setDropSnapshot(AUTOMATED_NAME_PREFIX); + GlobalStateMgr.getCurrentState().getEditLog().logClusterSnapshotLog(log); + } + + protected void setAutomatedSnapshotOff() { + // drop AUTOMATED snapshot + automatedSnapshotSvName = ""; + } + + public void replayLog(ClusterSnapshotLog log) { + ClusterSnapshotLog.ClusterSnapshotLogType logType = log.getType(); + switch (logType) { + case CREATE_SNAPSHOT_PREFIX: { + String createSnapshotNamePrefix = log.getCreateSnapshotNamePrefix(); + String storageVolumeName = log.getStorageVolumeName(); + if (createSnapshotNamePrefix.equals(AUTOMATED_NAME_PREFIX)) { + setAutomatedSnapshotOn(storageVolumeName); + } + break; + } + case DROP_SNAPSHOT: { + String dropSnapshotName = log.getDropSnapshotName(); + if (dropSnapshotName.equals(AUTOMATED_NAME_PREFIX)) { + setAutomatedSnapshotOff(); + } + break; + } + default: { + LOG.warn("Invalid Cluster Snapshot Log Type {}", logType); + } + } + } + + public void save(ImageWriter imageWriter) throws IOException, SRMetaBlockException { + SRMetaBlockWriter writer = imageWriter.getBlockWriter(SRMetaBlockID.CLUSTER_SNAPSHOT_MGR, 1); + writer.writeJson(this); + writer.close(); + } + + public void load(SRMetaBlockReader reader) + throws SRMetaBlockEOFException, IOException, SRMetaBlockException { + ClusterSnapshotMgr data = reader.readJson(ClusterSnapshotMgr.class); + } + + @Override + public void gsonPostProcess() throws IOException { + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/ClusterSnapshotLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/ClusterSnapshotLog.java new file mode 100644 index 0000000000000..e62d116e6d18b --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/persist/ClusterSnapshotLog.java @@ -0,0 +1,78 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.persist; + +import com.google.gson.annotations.SerializedName; +import com.starrocks.common.io.Text; +import com.starrocks.common.io.Writable; +import com.starrocks.persist.gson.GsonUtils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class ClusterSnapshotLog implements Writable { + public enum ClusterSnapshotLogType { NONE, CREATE_SNAPSHOT_PREFIX, DROP_SNAPSHOT } + @SerializedName(value = "type") + private ClusterSnapshotLogType type = ClusterSnapshotLogType.NONE; + // For CREATE_SNAPSHOT_PREFIX + @SerializedName(value = "createSnapshotNamePrefix") + private String createSnapshotNamePrefix = ""; + @SerializedName(value = "storageVolumeName") + private String storageVolumeName = ""; + // For DROP_SNAPSHOT + @SerializedName(value = "dropSnapshotName") + private String dropSnapshotName = ""; + + public ClusterSnapshotLog() {} + + public void setCreateSnapshotNamePrefix(String createSnapshotNamePrefix, String storageVolumeName) { + this.type = ClusterSnapshotLogType.CREATE_SNAPSHOT_PREFIX; + this.createSnapshotNamePrefix = createSnapshotNamePrefix; + this.storageVolumeName = storageVolumeName; + } + + public void setDropSnapshot(String dropSnapshotName) { + this.type = ClusterSnapshotLogType.DROP_SNAPSHOT; + this.dropSnapshotName = dropSnapshotName; + } + + public ClusterSnapshotLogType getType() { + return type; + } + + public String getCreateSnapshotNamePrefix() { + return this.createSnapshotNamePrefix; + } + + public String getStorageVolumeName() { + return this.storageVolumeName; + } + + public String getDropSnapshotName() { + return this.dropSnapshotName; + } + + public static ClusterSnapshotLog read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, ClusterSnapshotLog.class); + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java index a7eb1428676cf..c6d42ee3b1f11 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java @@ -1110,6 +1110,11 @@ public void loadJournal(GlobalStateMgr globalStateMgr, JournalEntity journal) warehouseMgr.replayAlterWarehouse(wh); break; } + case OperationType.OP_CLUSTER_SNAPSHOT_LOG: { + ClusterSnapshotLog log = (ClusterSnapshotLog) journal.getData(); + globalStateMgr.getClusterSnapshotMgr().replayLog(log); + break; + } default: { if (Config.metadata_ignore_unknown_operation_type) { LOG.warn("UNKNOWN Operation Type {}", opCode); @@ -1957,4 +1962,8 @@ public void logCancelDisableDisk(CancelDisableDiskInfo info) { public void logRecoverPartitionVersion(PartitionVersionRecoveryInfo info) { logEdit(OperationType.OP_RECOVER_PARTITION_VERSION, info); } + + public void logClusterSnapshotLog(ClusterSnapshotLog info) { + logEdit(OperationType.OP_CLUSTER_SNAPSHOT_LOG, info); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java index aa94a62600343..892150ae323d5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java @@ -588,6 +588,9 @@ public class OperationType { @IgnorableOnReplayFailed public static final short OP_ADD_KEY = 13512; + @IgnorableOnReplayFailed + public static final short OP_CLUSTER_SNAPSHOT_LOG = 13513; + /** * NOTICE: OperationType cannot use a value exceeding 20000, please follow the above sequence number */ diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java index f46283654c213..f9a7505b9f31a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java @@ -97,6 +97,8 @@ public int getId() { public static final SRMetaBlockID PIPE_MGR = new SRMetaBlockID(32); + public static final SRMetaBlockID CLUSTER_SNAPSHOT_MGR = new SRMetaBlockID(33); + /** * NOTICE: SRMetaBlockID cannot use a value exceeding 20000, please follow the above sequence number */ diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java index 1de93fcd2657b..d99d5d1929c63 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java @@ -41,6 +41,8 @@ import com.starrocks.sql.ast.AdminCancelRepairTableStmt; import com.starrocks.sql.ast.AdminCheckTabletsStmt; import com.starrocks.sql.ast.AdminRepairTableStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOffStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOnStmt; import com.starrocks.sql.ast.AdminSetConfigStmt; import com.starrocks.sql.ast.AdminSetPartitionVersionStmt; import com.starrocks.sql.ast.AdminSetReplicaStatusStmt; @@ -1174,6 +1176,36 @@ public ShowResultSet visitDropWarehouseStatement(DropWarehouseStmt stmt, Connect }); return null; } +<<<<<<< HEAD +======= + + @Override + public ShowResultSet visitAlterWarehouseStatement(AlterWarehouseStmt stmt, ConnectContext context) { + ErrorReport.wrapWithRuntimeException(() -> { + WarehouseManager warehouseMgr = context.getGlobalStateMgr().getWarehouseMgr(); + warehouseMgr.alterWarehouse(stmt); + }); + return null; + } + + @Override + public ShowResultSet visitAdminSetAutomatedSnapshotOnStatement(AdminSetAutomatedSnapshotOnStmt stmt, + ConnectContext context) { + ErrorReport.wrapWithRuntimeException(() -> { + context.getGlobalStateMgr().getClusterSnapshotMgr().setAutomatedSnapshotOn(stmt); + }); + return null; + } + + @Override + public ShowResultSet visitAdminSetAutomatedSnapshotOffStatement(AdminSetAutomatedSnapshotOffStmt stmt, + ConnectContext context) { + ErrorReport.wrapWithRuntimeException(() -> { + context.getGlobalStateMgr().getClusterSnapshotMgr().setAutomatedSnapshotOff(stmt); + }); + return null; + } +>>>>>>> be70af0d8 ([Feature] Support Cluster Snapshot Backup: SQL Interface and meta data (part 1) (#54447)) } } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index 01681567031c3..d5f174accd7d7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -139,6 +139,7 @@ import com.starrocks.lake.StarOSAgent; import com.starrocks.lake.compaction.CompactionControlScheduler; import com.starrocks.lake.compaction.CompactionMgr; +import com.starrocks.lake.snapshot.ClusterSnapshotMgr; import com.starrocks.lake.vacuum.AutovacuumDaemon; import com.starrocks.leader.CheckpointController; import com.starrocks.leader.TaskRunStateSynchronizer; @@ -514,6 +515,8 @@ public class GlobalStateMgr { private final ExecutorService queryDeployExecutor; private final WarehouseIdleChecker warehouseIdleChecker; + private final ClusterSnapshotMgr clusterSnapshotMgr; + public NodeMgr getNodeMgr() { return nodeMgr; } @@ -814,7 +817,10 @@ public void transferToNonLeader(FrontendNodeType newType) { this.queryDeployExecutor = ThreadPoolManager.newDaemonFixedThreadPool(Config.query_deploy_threadpool_size, Integer.MAX_VALUE, "query-deploy", true); + this.warehouseIdleChecker = new WarehouseIdleChecker(); + + this.clusterSnapshotMgr = new ClusterSnapshotMgr(); } public static void destroyCheckpoint() { @@ -1057,6 +1063,10 @@ public GlobalConstraintManager getGlobalConstraintManager() { return globalConstraintManager; } + public ClusterSnapshotMgr getClusterSnapshotMgr() { + return clusterSnapshotMgr; + } + // Use tryLock to avoid potential deadlock public boolean tryLock(boolean mustLock) { while (true) { @@ -1541,6 +1551,7 @@ public void loadImage(String imageDir) throws IOException { .put(SRMetaBlockID.KEY_MGR, keyMgr::load) .put(SRMetaBlockID.PIPE_MGR, pipeManager.getRepo()::load) .put(SRMetaBlockID.WAREHOUSE_MGR, warehouseMgr::load) + .put(SRMetaBlockID.CLUSTER_SNAPSHOT_MGR, clusterSnapshotMgr::load) .build(); Set metaMgrMustExists = new HashSet<>(loadImages.keySet()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/Analyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/Analyzer.java index 7275406149372..9467ef3191321 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/Analyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/Analyzer.java @@ -21,6 +21,8 @@ import com.starrocks.sql.ast.AdminCancelRepairTableStmt; import com.starrocks.sql.ast.AdminCheckTabletsStmt; import com.starrocks.sql.ast.AdminRepairTableStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOffStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOnStmt; import com.starrocks.sql.ast.AdminSetConfigStmt; import com.starrocks.sql.ast.AdminSetPartitionVersionStmt; import com.starrocks.sql.ast.AdminSetReplicaStatusStmt; @@ -592,6 +594,19 @@ public Void visitPauseRoutineLoadStatement(PauseRoutineLoadStmt statement, Conne return null; } + @Override + public Void visitAdminSetAutomatedSnapshotOnStatement(AdminSetAutomatedSnapshotOnStmt statement, ConnectContext context) { + ClusterSnapshotAnalyzer.analyze(statement, context); + return null; + } + + @Override + public Void visitAdminSetAutomatedSnapshotOffStatement(AdminSetAutomatedSnapshotOffStmt statement, + ConnectContext context) { + ClusterSnapshotAnalyzer.analyze(statement, context); + return null; + } + // ---------------------------------------- Catalog Statement ------------------------------------------- @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ClusterSnapshotAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ClusterSnapshotAnalyzer.java new file mode 100644 index 0000000000000..c643c956add60 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ClusterSnapshotAnalyzer.java @@ -0,0 +1,62 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.analyzer; + +import com.starrocks.common.DdlException; +import com.starrocks.qe.ConnectContext; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.server.StorageVolumeMgr; +import com.starrocks.sql.analyzer.SemanticException; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOffStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOnStmt; +import com.starrocks.sql.ast.AstVisitor; +import com.starrocks.sql.ast.StatementBase; + +public class ClusterSnapshotAnalyzer { + public static void analyze(StatementBase stmt, ConnectContext session) { + new ClusterSnapshotAnalyzer.ClusterSnapshotAnalyzerVisitor().visit(stmt, session); + } + + static class ClusterSnapshotAnalyzerVisitor implements AstVisitor { + @Override + public Void visitAdminSetAutomatedSnapshotOnStatement(AdminSetAutomatedSnapshotOnStmt statement, ConnectContext context) { + if (GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().isAutomatedSnapshotOn()) { + throw new SemanticException("Automated snapshot has been turn on"); + } + + String storageVolumeName = statement.getStorageVolumeName(); + StorageVolumeMgr storageVolumeMgr = GlobalStateMgr.getCurrentState().getStorageVolumeMgr(); + try { + if (!storageVolumeMgr.exists(storageVolumeName)) { + throw new SemanticException("Unknown storage volume: %s", storageVolumeName); + } + } catch (DdlException e) { + throw new SemanticException("Failed to get storage volume", e); + } + + return null; + } + + @Override + public Void visitAdminSetAutomatedSnapshotOffStatement(AdminSetAutomatedSnapshotOffStmt statement, + ConnectContext context) { + if (!GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().isAutomatedSnapshotOn()) { + throw new SemanticException("Automated snapshot has not been turn on"); + } + + return null; + } + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOffStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOffStmt.java new file mode 100644 index 0000000000000..3eafc0edd9c1e --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOffStmt.java @@ -0,0 +1,33 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.ast; + +import com.starrocks.sql.parser.NodePosition; + +public class AdminSetAutomatedSnapshotOffStmt extends DdlStmt { + + public AdminSetAutomatedSnapshotOffStmt() { + super(NodePosition.ZERO); + } + + public AdminSetAutomatedSnapshotOffStmt(NodePosition pos) { + super(pos); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitAdminSetAutomatedSnapshotOffStatement(this, context); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOnStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOnStmt.java new file mode 100644 index 0000000000000..b8f209ee75db4 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AdminSetAutomatedSnapshotOnStmt.java @@ -0,0 +1,40 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.ast; + +import com.starrocks.sql.parser.NodePosition; + +public class AdminSetAutomatedSnapshotOnStmt extends DdlStmt { + private final String storageVolumeName; + + public AdminSetAutomatedSnapshotOnStmt(String storageVolumeName) { + super(NodePosition.ZERO); + this.storageVolumeName = storageVolumeName; + } + + public AdminSetAutomatedSnapshotOnStmt(String storageVolumeName, NodePosition pos) { + super(pos); + this.storageVolumeName = storageVolumeName; + } + + public String getStorageVolumeName() { + return storageVolumeName; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitAdminSetAutomatedSnapshotOnStatement(this, context); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AstVisitor.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AstVisitor.java index 68921a1188f2f..d2163eeee6300 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AstVisitor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AstVisitor.java @@ -421,6 +421,14 @@ default R visitSyncStatement(SyncStmt statement, C context) { return visitDDLStatement(statement, context); } + default R visitAdminSetAutomatedSnapshotOnStatement(AdminSetAutomatedSnapshotOnStmt clause, C context) { + return visitDDLStatement(clause, context); + } + + default R visitAdminSetAutomatedSnapshotOffStatement(AdminSetAutomatedSnapshotOffStmt clause, C context) { + return visitDDLStatement(clause, context); + } + // ---------------------------------------- Cluster Management Statement ------------------------------------------- default R visitAlterSystemStatement(AlterSystemStmt statement, C context) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 8ff92e1e8eebb..90703e39b9488 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -114,6 +114,7 @@ import com.starrocks.qe.OriginStatement; import com.starrocks.qe.SqlModeHelper; import com.starrocks.scheduler.persist.TaskSchedule; +import com.starrocks.server.StorageVolumeMgr; import com.starrocks.server.WarehouseManager; import com.starrocks.sql.ShowTemporaryTableStmt; import com.starrocks.sql.analyzer.AnalyzerUtils; @@ -136,6 +137,8 @@ import com.starrocks.sql.ast.AdminCancelRepairTableStmt; import com.starrocks.sql.ast.AdminCheckTabletsStmt; import com.starrocks.sql.ast.AdminRepairTableStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOffStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOnStmt; import com.starrocks.sql.ast.AdminSetConfigStmt; import com.starrocks.sql.ast.AdminSetPartitionVersionStmt; import com.starrocks.sql.ast.AdminSetReplicaStatusStmt; @@ -2596,6 +2599,22 @@ public ParseNode visitSyncStatement(StarRocksParser.SyncStatementContext context return new SyncStmt(createPos(context)); } + @Override + public ParseNode visitAdminSetAutomatedSnapshotOnStatement( + StarRocksParser.AdminSetAutomatedSnapshotOnStatementContext context) { + String svName = StorageVolumeMgr.BUILTIN_STORAGE_VOLUME; + if (context.svName != null) { + svName = getIdentifierName(context.svName); + } + return new AdminSetAutomatedSnapshotOnStmt(svName, createPos(context)); + } + + @Override + public ParseNode visitAdminSetAutomatedSnapshotOffStatement( + StarRocksParser.AdminSetAutomatedSnapshotOffStatementContext context) { + return new AdminSetAutomatedSnapshotOffStmt(createPos(context)); + } + // ------------------------------------------- Cluster Management Statement ---------------------------------------- @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index 58223a471aa51..19fea3d1036fa 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -126,6 +126,8 @@ statement | killStatement | syncStatement | executeScriptStatement + | adminSetAutomatedSnapshotOnStatement + | adminSetAutomatedSnapshotOffStatement // Cluster Management Statement | alterSystemStatement @@ -725,6 +727,14 @@ syncStatement : SYNC ; +adminSetAutomatedSnapshotOnStatement + : ADMIN SET AUTOMATED CLUSTER SNAPSHOT ON (STORAGE VOLUME svName=identifier)? + ; + +adminSetAutomatedSnapshotOffStatement + : ADMIN SET AUTOMATED CLUSTER SNAPSHOT OFF + ; + // ------------------------------------------- Cluster Management Statement --------------------------------------------- alterSystemStatement @@ -2869,7 +2879,7 @@ number ; nonReserved - : ACCESS | ACTIVE | ADVISOR | AFTER | AGGREGATE | APPLY | ASYNC | AUTHORS | AVG | ADMIN | ANTI | AUTHENTICATION | AUTO_INCREMENT + : ACCESS | ACTIVE | ADVISOR | AFTER | AGGREGATE | APPLY | ASYNC | AUTHORS | AVG | ADMIN | ANTI | AUTHENTICATION | AUTO_INCREMENT | AUTOMATED | ARRAY_AGG | ARRAY_AGG_DISTINCT | BACKEND | BACKENDS | BACKUP | BEGIN | BITMAP_UNION | BLACKLIST | BLACKHOLE | BINARY | BODY | BOOLEAN | BRANCH | BROKER | BUCKETS | BUILTIN | BASE | BEFORE @@ -2888,7 +2898,7 @@ nonReserved | LABEL | LAST | LESS | LEVEL | LIST | LOCAL | LOCATION | LOGS | LOGICAL | LOW_PRIORITY | LOCK | LOCATIONS | MANUAL | MAP | MAPPING | MAPPINGS | MASKING | MATCH | MAPPINGS | MATERIALIZED | MAX | META | MIN | MINUTE | MINUTES | MODE | MODIFY | MONTH | MERGE | MINUS | NAME | NAMES | NEGATIVE | NO | NODE | NODES | NONE | NULLS | NUMBER | NUMERIC - | OBSERVER | OF | OFFSET | ONLY | OPTIMIZER | OPEN | OPERATE | OPTION | OVERWRITE + | OBSERVER | OF | OFFSET | ONLY | OPTIMIZER | OPEN | OPERATE | OPTION | OVERWRITE | OFF | PARTITIONS | PASSWORD | PATH | PAUSE | PENDING | PERCENTILE_UNION | PIVOT | PLAN | PLUGIN | PLUGINS | POLICY | POLICIES | PERCENT_RANK | PRECEDING | PRIORITY | PROC | PROCESSLIST | PROFILE | PROFILELIST | PRIVILEGES | PROBABILITY | PROPERTIES | PROPERTY | PIPE | PIPES | QUARTER | QUERY | QUERIES | QUEUE | QUOTA | QUALIFY diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 index f318b29550cba..6d8abe7d3df86 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 @@ -45,6 +45,7 @@ ASC: 'ASC'; ASYNC: 'ASYNC'; AUTHORS: 'AUTHORS'; AUTHENTICATION: 'AUTHENTICATION'; +AUTOMATED: 'AUTOMATED'; AUTO_INCREMENT: 'AUTO_INCREMENT'; AVG: 'AVG'; BACKEND: 'BACKEND'; @@ -292,6 +293,7 @@ NUMBER: 'NUMBER'; NUMERIC: 'NUMERIC'; OBSERVER: 'OBSERVER'; OF: 'OF'; +OFF: 'OFF'; OFFSET: 'OFFSET'; ON: 'ON'; ONLY: 'ONLY'; diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java new file mode 100644 index 0000000000000..57c785e744d68 --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java @@ -0,0 +1,225 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.lake.snapshot; + +//import com.google.common.collect.Lists; +import com.starrocks.alter.AlterTest; +import com.starrocks.backup.BlobStorage; +import com.starrocks.backup.Status; +import com.starrocks.common.AlreadyExistsException; +import com.starrocks.common.DdlException; +import com.starrocks.common.MetaNotFoundException; +//import com.starrocks.common.Pair; +//import com.starrocks.journal.bdbje.BDBJEJournal; +import com.starrocks.lake.StarOSAgent; +//import com.starrocks.lake.snapshot.ClusterSnapshotJob.ClusterSnapshotJobState; +import com.starrocks.persist.ClusterSnapshotLog; +import com.starrocks.persist.EditLog; +import com.starrocks.persist.Storage; +import com.starrocks.server.GlobalStateMgr; +//import com.starrocks.server.RunMode; +import com.starrocks.server.StorageVolumeMgr; +import com.starrocks.sql.analyzer.AnalyzeTestUtil; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOffStmt; +import com.starrocks.sql.ast.AdminSetAutomatedSnapshotOnStmt; +import mockit.Delegate; +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_ENDPOINT; +import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_REGION; +import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_USE_AWS_SDK_DEFAULT_BEHAVIOR; +import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeFail; +import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeSuccess; + +public class ClusterSnapshotTest { + @Mocked + private EditLog editLog; + + private StarOSAgent starOSAgent = new StarOSAgent(); + + private String storageVolumeName = StorageVolumeMgr.BUILTIN_STORAGE_VOLUME; + private ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr(); + private boolean initSv = false; + + private File mockedFile = new File("/abc/abc"); + + @BeforeClass + public static void beforeClass() throws Exception { + AlterTest.beforeClass(); + AnalyzeTestUtil.init(); + } + + @Before + public void setUp() { + try { + initStorageVolume(); + } catch (Exception ignore) { + } + + new Expectations() { + { + editLog.logClusterSnapshotLog((ClusterSnapshotLog) any); + minTimes = 0; + result = new Delegate() { + public void logClusterSnapshotLog(ClusterSnapshotLog log) { + } + }; + } + }; + + new MockUp() { + @Mock + public EditLog getEditLog() { + return editLog; + } + + @Mock + public ClusterSnapshotMgr getClusterSnapshotMgr() { + return clusterSnapshotMgr; + } + + @Mock + public long getNextId() { + return 0L; + } + }; + + new MockUp() { + @Mock + public Status delete(String remotePath) { + return Status.OK; + } + }; + + new MockUp() { + @Mock + public File getCurrentImageFile() { + return mockedFile; + } + + @Mock + public File getCurrentChecksumFile() { + return mockedFile; + } + + @Mock + public File getRoleFile() { + return mockedFile; + } + + @Mock + public File getVersionFile() { + return mockedFile; + } + }; + + new MockUp() { + @Mock + public StarOSAgent getStarOSAgent() { + return starOSAgent; + } + }; + + new MockUp() { + @Mock + public String getRawServiceId() { + return "qwertty"; + } + }; + + setAutomatedSnapshotOff(false); + } + + private void setAutomatedSnapshotOn(boolean testReplay) { + if (!testReplay) { + GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().setAutomatedSnapshotOn( + new AdminSetAutomatedSnapshotOnStmt(storageVolumeName)); + } else { + GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().setAutomatedSnapshotOn(storageVolumeName); + } + } + + private void setAutomatedSnapshotOff(boolean testReplay) { + if (!testReplay) { + GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().setAutomatedSnapshotOff( + new AdminSetAutomatedSnapshotOffStmt()); + } else { + GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().setAutomatedSnapshotOff(); + } + } + + private void initStorageVolume() throws AlreadyExistsException, DdlException, MetaNotFoundException { + if (!initSv) { + List locations = Arrays.asList("s3://abc"); + Map storageParams = new HashMap<>(); + storageParams.put(AWS_S3_REGION, "region"); + storageParams.put(AWS_S3_ENDPOINT, "endpoint"); + storageParams.put(AWS_S3_USE_AWS_SDK_DEFAULT_BEHAVIOR, "true"); + String svKey = GlobalStateMgr.getCurrentState().getStorageVolumeMgr() + .createStorageVolume(storageVolumeName, "S3", locations, storageParams, Optional.empty(), ""); + Assert.assertEquals(true, GlobalStateMgr.getCurrentState().getStorageVolumeMgr().exists(storageVolumeName)); + Assert.assertEquals(storageVolumeName, + GlobalStateMgr.getCurrentState().getStorageVolumeMgr().getStorageVolumeName(svKey)); + initSv = true; + } + } + + @Test + public void testOperationOfAutomatedSnapshot() throws DdlException { + // 1. test analyer and execution + String turnOnSql = "ADMIN SET AUTOMATED CLUSTER SNAPSHOT ON"; + // no sv + analyzeFail(turnOnSql + " STORAGE VOLUME testSv"); + + analyzeSuccess(turnOnSql); + setAutomatedSnapshotOn(false); + // duplicate creation + analyzeFail(turnOnSql); + + setAutomatedSnapshotOff(false); + + String turnOFFSql = "ADMIN SET AUTOMATED CLUSTER SNAPSHOT OFF"; + analyzeFail(turnOFFSql); + setAutomatedSnapshotOn(false); + analyzeSuccess(turnOFFSql); + } + + @Test + public void testReplayClusterSnapshotLog() { + // create atuomated snapshot request log + ClusterSnapshotLog logCreate = new ClusterSnapshotLog(); + logCreate.setCreateSnapshotNamePrefix(ClusterSnapshotMgr.AUTOMATED_NAME_PREFIX, storageVolumeName); + GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().replayLog(logCreate); + + // drop automated snapshot request log + ClusterSnapshotLog logDrop = new ClusterSnapshotLog(); + logDrop.setDropSnapshot(ClusterSnapshotMgr.AUTOMATED_NAME_PREFIX); + GlobalStateMgr.getCurrentState().getClusterSnapshotMgr().replayLog(logDrop); + } +} diff --git a/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java b/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java index 4d75611d49494..8bf1b7cd69683 100644 --- a/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java @@ -151,6 +151,7 @@ public void testRecoverableOperations() { Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_REPLICATION_JOB)); Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_DELETE_REPLICATION_JOB)); Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_RESET_FRONTENDS)); + Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_CLUSTER_SNAPSHOT_LOG)); } @Test From f208d25de7d00cd09278c6cd85992d4c30a61bd7 Mon Sep 17 00:00:00 2001 From: srlch <111035020+srlch@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:23:48 +0800 Subject: [PATCH 2/2] fix conflict (#54504) Signed-off-by: srlch --- .../main/java/com/starrocks/qe/DDLStmtExecutor.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java index d99d5d1929c63..35616573a8b89 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/DDLStmtExecutor.java @@ -1176,17 +1176,6 @@ public ShowResultSet visitDropWarehouseStatement(DropWarehouseStmt stmt, Connect }); return null; } -<<<<<<< HEAD -======= - - @Override - public ShowResultSet visitAlterWarehouseStatement(AlterWarehouseStmt stmt, ConnectContext context) { - ErrorReport.wrapWithRuntimeException(() -> { - WarehouseManager warehouseMgr = context.getGlobalStateMgr().getWarehouseMgr(); - warehouseMgr.alterWarehouse(stmt); - }); - return null; - } @Override public ShowResultSet visitAdminSetAutomatedSnapshotOnStatement(AdminSetAutomatedSnapshotOnStmt stmt, @@ -1205,7 +1194,6 @@ public ShowResultSet visitAdminSetAutomatedSnapshotOffStatement(AdminSetAutomate }); return null; } ->>>>>>> be70af0d8 ([Feature] Support Cluster Snapshot Backup: SQL Interface and meta data (part 1) (#54447)) } }