diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java index a4c24de589130..9f8ab6ee824cd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java @@ -700,11 +700,9 @@ private void saveMetaInfo() { localMetaInfoFilePath = metaInfoFile.getAbsolutePath(); // 3. save job info file - // save table info into BackupJobInfo only for OlapTable or MV - List olapTbls = backupMeta.getTables().values().stream() - .filter(Table::isOlapTableOrMaterializedView).collect(Collectors.toList()); - jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, olapTbls, snapshotInfos); - LOG.warn("job info: {}. {}", jobInfo, this); + jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, backupMeta.getTables().values(), + snapshotInfos); + LOG.debug("job info: {}. {}", jobInfo, this); File jobInfoFile = new File(jobDir, Repository.PREFIX_JOB_INFO + createTimeStr); if (!jobInfoFile.createNewFile()) { status = new Status(ErrCode.COMMON_ERROR, "Failed to create job info file: " + jobInfoFile.toString()); diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java index cbef5f0cc0789..abb1c7cd91506 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java @@ -303,11 +303,16 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db // tbls for (Table tbl : tbls) { - OlapTable olapTbl = (OlapTable) tbl; BackupTableInfo tableInfo = new BackupTableInfo(); tableInfo.id = tbl.getId(); tableInfo.name = tbl.getName(); jobInfo.tables.put(tableInfo.name, tableInfo); + + if (tbl.isOlapView()) { + continue; + } + + OlapTable olapTbl = (OlapTable) tbl; // partitions for (Partition partition : olapTbl.getPartitions()) { BackupPartitionInfo partitionInfo = new BackupPartitionInfo(); @@ -450,6 +455,11 @@ private static void genFromJson(String json, BackupJobInfo jobInfo) { } JSONObject parts = tbl.getJSONObject("partitions"); String[] partsNames = JSONObject.getNames(parts); + if (partsNames == null) { + // skip logical view + jobInfo.tables.put(tblName, tblInfo); + continue; + } for (String partName : partsNames) { BackupPartitionInfo partInfo = new BackupPartitionInfo(); partInfo.name = partName; diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java index cbc4fc20101b3..a5cc9e7159057 100644 --- a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java @@ -46,7 +46,6 @@ import com.google.common.collect.Table.Cell; import com.google.gson.annotations.SerializedName; import com.starrocks.analysis.BrokerDesc; -import com.starrocks.analysis.TableName; import com.starrocks.backup.BackupJobInfo.BackupIndexInfo; import com.starrocks.backup.BackupJobInfo.BackupPartitionInfo; import com.starrocks.backup.BackupJobInfo.BackupPhysicalPartitionInfo; @@ -77,10 +76,8 @@ import com.starrocks.catalog.Table; import com.starrocks.catalog.Tablet; import com.starrocks.catalog.TabletMeta; -import com.starrocks.catalog.View; import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; -import com.starrocks.common.DdlException; import com.starrocks.common.MarkedCountDownLatch; import com.starrocks.common.Pair; import com.starrocks.common.UserException; @@ -90,10 +87,7 @@ import com.starrocks.fs.HdfsUtil; import com.starrocks.metric.MetricRepo; import com.starrocks.persist.ColocatePersistInfo; -import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; -import com.starrocks.sql.ast.CreateViewStmt; -import com.starrocks.sql.parser.NodePosition; import com.starrocks.task.AgentBatchTask; import com.starrocks.task.AgentTask; import com.starrocks.task.AgentTaskExecutor; @@ -526,7 +520,21 @@ private void checkAndPrepareMeta() { for (BackupTableInfo tblInfo : jobInfo.tables.values()) { Table remoteTbl = backupMeta.getTable(tblInfo.name); Preconditions.checkNotNull(remoteTbl); + Table localTbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); + + if (localTbl != null && remoteTbl.isOlapView() && !localTbl.isOlapView()) { + status = new Status(ErrCode.BAD_REPLACE, + "Table: " + localTbl.getName() + " has existed and it is not a View"); + return; + } + + if (remoteTbl.isOlapView()) { + remoteTbl.setId(globalStateMgr.getNextId()); + restoredTbls.add(remoteTbl); + continue; + } + if (localTbl != null) { if (localTbl instanceof OlapTable && localTbl.hasAutoIncrementColumn()) { // it must be !isReplay == true @@ -755,7 +763,7 @@ private void checkAndPrepareMeta() { backupTableInfo.getPartInfo(restorePart.getName()), true); } // set restored table's new name after all 'genFileMapping' - ((OlapTable) restoreTbl).setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); + restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); } LOG.debug("finished to generate create replica tasks. {}", this); @@ -775,14 +783,6 @@ private void checkAndPrepareMeta() { return; } - // add all restored olap view into globalStateMgr - List restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView) - .map(x -> (View) x).collect(Collectors.toList()); - addRestoreOlapView(restoredOlapViews); - if (!status.ok()) { - return; - } - LOG.info("finished to prepare meta. begin to make snapshot. {}", this); // begin to make snapshots for all replicas @@ -840,38 +840,6 @@ protected void sendCreateReplicaTasks() { } } - protected void addRestoreOlapView(List restoredOlapViews) { - Database db = globalStateMgr.getLocalMetastore().getDb(dbId); - - ConnectContext context = new ConnectContext(); - context.setDatabase(db.getFullName()); - context.setGlobalStateMgr(globalStateMgr); - context.setStartTime(); - context.setThreadLocalInfo(); - - for (View restoredOlapView : restoredOlapViews) { - Table localTbl = db.getTable(restoredOlapView.getId()); - if (localTbl != null && !localTbl.isOlapView()) { - status = new Status(ErrCode.BAD_REPLACE, - "Table: " + localTbl.getName() + " has existed and it is not a View"); - return; - } - - CreateViewStmt stmt = new CreateViewStmt(false, true, new TableName(db.getFullName(), restoredOlapView.getName()), - Lists.newArrayList(), restoredOlapView.getComment(), restoredOlapView.getQueryStatement(), NodePosition.ZERO); - stmt.setColumns(restoredOlapView.getColumns()); - stmt.setInlineViewDef(restoredOlapView.getInlineViewDef()); - context.getSessionVariable().setSqlMode(restoredOlapView.getSqlMode()); - try { - GlobalStateMgr.getCurrentState().createView(stmt); - } catch (DdlException e) { - status = new Status(ErrCode.COMMON_ERROR, - "Failed to create view for restore. err message: " + e.getMessage()); - return; - } - } - } - protected void addRestorePartitionsAndTables(Database db) { db.writeLock(); try { @@ -882,6 +850,10 @@ protected void addRestorePartitionsAndTables(Database db) { // add restored tables for (Table tbl : restoredTbls) { + if (tbl.isOlapView()) { + // for logical view, force drop and replace + db.dropTable(tbl.getName()); + } if (!db.registerTableUnlocked(tbl)) { status = new Status(ErrCode.COMMON_ERROR, "Table " + tbl.getName() + " already exist in db: " + db.getOriginName()); @@ -1131,7 +1103,7 @@ private void replayCheckAndPrepareMeta() { // replay set all existing tables's state to RESTORE for (BackupTableInfo tblInfo : jobInfo.tables.values()) { Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name)); - if (tbl == null) { + if (tbl == null || tbl.isOlapView()) { continue; } OlapTable olapTbl = (OlapTable) tbl; @@ -1153,9 +1125,8 @@ private void replayCheckAndPrepareMeta() { db.writeUnlock(); } - List restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView) - .map(x -> (View) x).collect(Collectors.toList()); - addRestoreOlapView(restoredOlapViews); + // restored view need not to be added again here, because + // another edit log created by createView will done. LOG.info("replay check and prepare meta. {}", this); } @@ -1467,7 +1438,7 @@ private Status allTabletCommitted(boolean isReplay) { for (long tblId : restoredVersionInfo.rowKeySet()) { Table tbl = db.getTable(tblId); - if (tbl == null) { + if (tbl == null || tbl.isOlapView()) { continue; } OlapTable olapTbl = (OlapTable) tbl; diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java index 6f7dfdfa7282f..bbccd3f9db4fa 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java @@ -708,7 +708,7 @@ public void testColocateRestore() { public void testRestoreView() { new Expectations() { { - globalStateMgr.getLocalMetastore().getDb(anyLong); + globalStateMgr.getDb(anyLong); minTimes = 0; result = db; @@ -769,6 +769,11 @@ boolean await(long timeout, TimeUnit unit) { View restoredView = (View) db.getTable(CatalogMocker.TEST_TBL6_ID); + BackupTableInfo tblInfo = new BackupTableInfo(); + tblInfo.id = CatalogMocker.TEST_TBL6_ID; + tblInfo.name = CatalogMocker.TEST_TBL6_NAME; + jobInfo.tables.put(tblInfo.name, tblInfo); + new MockUp() { @Mock public Database getDb(String dbName) {