[BugFix] Fix several problem for logical view restore (backport #52291)

Signed-off-by: srlch <[email protected]>
StarRocks · Oct 29, 2024 · 5923e38 · 5923e38
1 parent 848dce7
commit 5923e38
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 58 deletions.
diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJob.java
@@ -700,11 +700,9 @@ private void saveMetaInfo() {
             localMetaInfoFilePath = metaInfoFile.getAbsolutePath();
 
             // 3. save job info file
-            // save table info into BackupJobInfo only for OlapTable or MV
-            List<Table> olapTbls = backupMeta.getTables().values().stream()
-                                   .filter(Table::isOlapTableOrMaterializedView).collect(Collectors.toList());
-            jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, olapTbls, snapshotInfos);
-            LOG.warn("job info: {}. {}", jobInfo, this);
+            jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, backupMeta.getTables().values(),
+                    snapshotInfos);
+            LOG.debug("job info: {}. {}", jobInfo, this);
             File jobInfoFile = new File(jobDir, Repository.PREFIX_JOB_INFO + createTimeStr);
             if (!jobInfoFile.createNewFile()) {
                 status = new Status(ErrCode.COMMON_ERROR, "Failed to create job info file: " + jobInfoFile.toString());

diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/com/starrocks/backup/BackupJobInfo.java
@@ -303,11 +303,16 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db
 
         // tbls
         for (Table tbl : tbls) {
-            OlapTable olapTbl = (OlapTable) tbl;
             BackupTableInfo tableInfo = new BackupTableInfo();
             tableInfo.id = tbl.getId();
             tableInfo.name = tbl.getName();
             jobInfo.tables.put(tableInfo.name, tableInfo);
+
+            if (tbl.isOlapView()) {
+                continue;
+            }
+
+            OlapTable olapTbl = (OlapTable) tbl;
             // partitions
             for (Partition partition : olapTbl.getPartitions()) {
                 BackupPartitionInfo partitionInfo = new BackupPartitionInfo();
@@ -450,6 +455,11 @@ private static void genFromJson(String json, BackupJobInfo jobInfo) {
             }
             JSONObject parts = tbl.getJSONObject("partitions");
             String[] partsNames = JSONObject.getNames(parts);
+            if (partsNames == null) {
+                // skip logical view
+                jobInfo.tables.put(tblName, tblInfo);
+                continue;
+            }
             for (String partName : partsNames) {
                 BackupPartitionInfo partInfo = new BackupPartitionInfo();
                 partInfo.name = partName;

diff --git a/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java b/fe/fe-core/src/main/java/com/starrocks/backup/RestoreJob.java
@@ -46,7 +46,6 @@
 import com.google.common.collect.Table.Cell;
 import com.google.gson.annotations.SerializedName;
 import com.starrocks.analysis.BrokerDesc;
-import com.starrocks.analysis.TableName;
 import com.starrocks.backup.BackupJobInfo.BackupIndexInfo;
 import com.starrocks.backup.BackupJobInfo.BackupPartitionInfo;
 import com.starrocks.backup.BackupJobInfo.BackupPhysicalPartitionInfo;
@@ -77,10 +76,8 @@
 import com.starrocks.catalog.Table;
 import com.starrocks.catalog.Tablet;
 import com.starrocks.catalog.TabletMeta;
-import com.starrocks.catalog.View;
 import com.starrocks.common.AnalysisException;
 import com.starrocks.common.Config;
-import com.starrocks.common.DdlException;
 import com.starrocks.common.MarkedCountDownLatch;
 import com.starrocks.common.Pair;
 import com.starrocks.common.UserException;
@@ -90,10 +87,7 @@
 import com.starrocks.fs.HdfsUtil;
 import com.starrocks.metric.MetricRepo;
 import com.starrocks.persist.ColocatePersistInfo;
-import com.starrocks.qe.ConnectContext;
 import com.starrocks.server.GlobalStateMgr;
-import com.starrocks.sql.ast.CreateViewStmt;
-import com.starrocks.sql.parser.NodePosition;
 import com.starrocks.task.AgentBatchTask;
 import com.starrocks.task.AgentTask;
 import com.starrocks.task.AgentTaskExecutor;
@@ -526,7 +520,21 @@ private void checkAndPrepareMeta() {
             for (BackupTableInfo tblInfo : jobInfo.tables.values()) {
                 Table remoteTbl = backupMeta.getTable(tblInfo.name);
                 Preconditions.checkNotNull(remoteTbl);
+
                 Table localTbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name));
+
+                if (localTbl != null && remoteTbl.isOlapView() && !localTbl.isOlapView()) {
+                    status = new Status(ErrCode.BAD_REPLACE,
+                                        "Table: " + localTbl.getName() + " has existed and it is not a View");
+                    return;
+                }
+
+                if (remoteTbl.isOlapView()) {
+                    remoteTbl.setId(globalStateMgr.getNextId());
+                    restoredTbls.add(remoteTbl);
+                    continue;
+                }
+
                 if (localTbl != null) {
                     if (localTbl instanceof OlapTable && localTbl.hasAutoIncrementColumn()) {
                         // it must be !isReplay == true
@@ -755,7 +763,7 @@ private void checkAndPrepareMeta() {
                             backupTableInfo.getPartInfo(restorePart.getName()), true);
                 }
                 // set restored table's new name after all 'genFileMapping'
-                ((OlapTable) restoreTbl).setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName()));
+                restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName()));
             }
 
             LOG.debug("finished to generate create replica tasks. {}", this);
@@ -775,14 +783,6 @@ private void checkAndPrepareMeta() {
             return;
         }
 
-        // add all restored olap view into globalStateMgr
-        List<View> restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView)
-                                       .map(x -> (View) x).collect(Collectors.toList());
-        addRestoreOlapView(restoredOlapViews);
-        if (!status.ok()) {
-            return;
-        }
-
         LOG.info("finished to prepare meta. begin to make snapshot. {}", this);
 
         // begin to make snapshots for all replicas
@@ -840,38 +840,6 @@ protected void sendCreateReplicaTasks() {
         }
     }
 
-    protected void addRestoreOlapView(List<View> restoredOlapViews) {
-        Database db = globalStateMgr.getLocalMetastore().getDb(dbId);
-
-        ConnectContext context = new ConnectContext();
-        context.setDatabase(db.getFullName());
-        context.setGlobalStateMgr(globalStateMgr);
-        context.setStartTime();
-        context.setThreadLocalInfo();
-
-        for (View restoredOlapView : restoredOlapViews) {
-            Table localTbl = db.getTable(restoredOlapView.getId());
-            if (localTbl != null && !localTbl.isOlapView()) {
-                status = new Status(ErrCode.BAD_REPLACE,
-                                    "Table: " + localTbl.getName() + " has existed and it is not a View");
-                return;
-            }
-
-            CreateViewStmt stmt = new CreateViewStmt(false, true, new TableName(db.getFullName(), restoredOlapView.getName()),
-                    Lists.newArrayList(), restoredOlapView.getComment(), restoredOlapView.getQueryStatement(), NodePosition.ZERO);
-            stmt.setColumns(restoredOlapView.getColumns());
-            stmt.setInlineViewDef(restoredOlapView.getInlineViewDef());
-            context.getSessionVariable().setSqlMode(restoredOlapView.getSqlMode());
-            try {
-                GlobalStateMgr.getCurrentState().createView(stmt);
-            } catch (DdlException e) {
-                status = new Status(ErrCode.COMMON_ERROR,
-                                    "Failed to create view for restore. err message: " + e.getMessage());
-                return;
-            }
-        }
-    }
-
     protected void addRestorePartitionsAndTables(Database db) {
         db.writeLock();
         try {
@@ -882,6 +850,10 @@ protected void addRestorePartitionsAndTables(Database db) {
 
             // add restored tables
             for (Table tbl : restoredTbls) {
+                if (tbl.isOlapView()) {
+                    // for logical view, force drop and replace
+                    db.dropTable(tbl.getName());
+                }
                 if (!db.registerTableUnlocked(tbl)) {
                     status = new Status(ErrCode.COMMON_ERROR, "Table " + tbl.getName()
                             + " already exist in db: " + db.getOriginName());
@@ -1131,7 +1103,7 @@ private void replayCheckAndPrepareMeta() {
             // replay set all existing tables's state to RESTORE
             for (BackupTableInfo tblInfo : jobInfo.tables.values()) {
                 Table tbl = db.getTable(jobInfo.getAliasByOriginNameIfSet(tblInfo.name));
-                if (tbl == null) {
+                if (tbl == null || tbl.isOlapView()) {
                     continue;
                 }
                 OlapTable olapTbl = (OlapTable) tbl;
@@ -1153,9 +1125,8 @@ private void replayCheckAndPrepareMeta() {
             db.writeUnlock();
         }
 
-        List<View> restoredOlapViews = backupMeta.getTables().values().stream().filter(Table::isOlapView)
-                                       .map(x -> (View) x).collect(Collectors.toList());
-        addRestoreOlapView(restoredOlapViews);
+        // restored view need not to be added again here, because
+        // another edit log created by createView will done.
 
         LOG.info("replay check and prepare meta. {}", this);
     }
@@ -1467,7 +1438,7 @@ private Status allTabletCommitted(boolean isReplay) {
 
             for (long tblId : restoredVersionInfo.rowKeySet()) {
                 Table tbl = db.getTable(tblId);
-                if (tbl == null) {
+                if (tbl == null || tbl.isOlapView()) {
                     continue;
                 }
                 OlapTable olapTbl = (OlapTable) tbl;

diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java
@@ -769,6 +769,11 @@ boolean await(long timeout, TimeUnit unit) {
 
         View restoredView = (View) db.getTable(CatalogMocker.TEST_TBL6_ID);
 
+        BackupTableInfo tblInfo = new BackupTableInfo();
+        tblInfo.id = CatalogMocker.TEST_TBL6_ID;
+        tblInfo.name = CatalogMocker.TEST_TBL6_NAME;
+        jobInfo.tables.put(tblInfo.name, tblInfo);
+
         new MockUp<LocalMetastore>() {
             @Mock
             public Database getDb(String dbName) {