From 9ed6e142e314ff9bda675bdc7cc6667366782e25 Mon Sep 17 00:00:00 2001 From: meegoo Date: Sun, 27 Oct 2024 22:17:11 -0700 Subject: [PATCH 1/3] =?UTF-8?q?[BugFix]=20Fix=20tablet=20meta=20use=20tabl?= =?UTF-8?q?etMeta=20uses=20partition=5Fid=20and=20physica=E2=80=A6=20(#523?= =?UTF-8?q?73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: meegoo (cherry picked from commit bf04f84df6f8ebac920ac6b3404a342976441330) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java # fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java # fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java # fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java # fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java # fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java # fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java # fe/fe-core/src/main/java/com/starrocks/lake/Utils.java # fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java # fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java # fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java --- .../starrocks/alter/SchemaChangeJobV2.java | 2 +- .../starrocks/catalog/CatalogRecycleBin.java | 8 + .../java/com/starrocks/catalog/Partition.java | 30 + .../catalog/TabletInvertedIndex.java | 18 +- .../com/starrocks/catalog/TabletMeta.java | 19 +- .../clone/ColocateTableBalancer.java | 2 +- .../clone/DiskAndTabletLoadReBalancer.java | 33 +- .../com/starrocks/clone/TabletChecker.java | 44 +- .../com/starrocks/clone/TabletSchedCtx.java | 26 +- .../com/starrocks/clone/TabletScheduler.java | 51 +- .../common/proc/StatisticProcDir.java | 6 +- .../consistency/CheckConsistencyJob.java | 44 +- .../consistency/ConsistencyChecker.java | 59 +- .../main/java/com/starrocks/lake/Utils.java | 5 + .../java/com/starrocks/leader/LeaderImpl.java | 2 +- .../com/starrocks/leader/ReportHandler.java | 29 +- .../persist/ConsistencyCheckInfo.java | 14 +- .../java/com/starrocks/qe/ShowExecutor.java | 527 ++++++++++++++++++ .../scheduler/mv/BinlogConsumeStateVO.java | 2 +- .../com/starrocks/server/LocalMetastore.java | 25 +- .../TabletMetadataUpdateAgentTaskFactory.java | 2 +- .../LakeTableTxnStateListener.java | 4 +- .../OlapTableTxnStateListener.java | 2 +- .../starrocks/catalog/TempPartitionTest.java | 2 +- .../starrocks/leader/ReportHandlerTest.java | 4 +- 25 files changed, 834 insertions(+), 126 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java index 9b5ce2d018d15..64c89e9a5f129 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java @@ -825,7 +825,7 @@ private void onFinished(OlapTable tbl) { } // replace the origin index with shadow index, set index state as NORMAL for (Partition partition : tbl.getPartitions()) { - TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium(); + TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getId()).getStorageMedium(); // drop the origin index from partitions for (Map.Entry entry : indexIdMap.entrySet()) { long shadowIdxId = entry.getKey(); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java index 21c1ccf38dd4c..29738a2b44c19 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java @@ -297,6 +297,14 @@ public synchronized List getPartitions(long tableId) { .collect(Collectors.toList()); } + public synchronized List getPhysicalPartitions(long tableId) { + return idToPartition.values().stream() + .filter(v -> (v.getTableId() == tableId)) + .map(RecyclePartitionInfo::getPartition) + .flatMap(p -> p.getSubPartitions().stream()) + .collect(Collectors.toList()); + } + /** * if we can erase this instance, we should check if anyone enable erase later. * Only used by main loop. diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java index 356d81590d308..ae5c36df6b589 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java @@ -648,4 +648,34 @@ public long getMinRetainVersion() { public void setMinRetainVersion(long minRetainVersion) { this.minRetainVersion = minRetainVersion; } +<<<<<<< HEAD +======= + + public String generatePhysicalPartitionName(long physicalPartitionId) { + return this.name + '_' + physicalPartitionId; + } + + @Override + public void gsonPostProcess() throws IOException { + if (dataVersion == 0) { + dataVersion = visibleVersion; + } + if (nextDataVersion == 0) { + nextDataVersion = nextVersion; + } + if (versionEpoch == 0) { + versionEpoch = nextVersionEpoch(); + } + if (versionTxnType == null) { + versionTxnType = TransactionType.TXN_NORMAL; + } + + for (PhysicalPartitionImpl subPartition : idToSubPartition.values()) { + if (subPartition.getName() == null) { + subPartition.setName(generatePhysicalPartitionName(subPartition.getId())); + } + nameToSubPartition.put(subPartition.getName(), subPartition); + } + } +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java index 8761fdf9460b0..b21a446a99b3e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java @@ -211,7 +211,7 @@ public void tabletReport(long backendId, Map backendTablets, replica.setLastReportVersion(backendTabletInfo.getVersion()); // check if tablet needs migration - long partitionId = tabletMeta.getPartitionId(); + long partitionId = tabletMeta.getPhysicalPartitionId(); TStorageMedium storageMedium = storageMediumMap.get(partitionId); if (storageMedium != null && backendTabletInfo.isSetStorage_medium()) { if (storageMedium != backendTabletInfo.getStorage_medium()) { @@ -239,7 +239,7 @@ public void tabletReport(long backendId, Map backendTablets, transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId); if (transactionState == null || transactionState.getTransactionStatus() == TransactionStatus.ABORTED) { - transactionsToClear.put(transactionId, tabletMeta.getPartitionId()); + transactionsToClear.put(transactionId, partitionId); LOG.debug("transaction id [{}] is not valid any more, " + "clear it from backend [{}]", transactionId, backendId); } else if (transactionState.getTransactionStatus() == @@ -266,7 +266,7 @@ public void tabletReport(long backendId, Map backendTablets, transactionState.getTransactionId()); } else { TPartitionVersionInfo versionInfo = - new TPartitionVersionInfo(tabletMeta.getPartitionId(), + new TPartitionVersionInfo(partitionId, partitionCommitInfo.getVersion(), 0); Map> txnMap = transactionsToPublish.computeIfAbsent( @@ -415,11 +415,11 @@ public void checkTabletMetaConsistency(Map creatingTableIds) { } // validate partition - long partitionId = tabletMeta.getPartitionId(); - PhysicalPartition partition = table.getPhysicalPartition(partitionId); - if (partition == null) { - partition = recycleBin.getPhysicalPartition(partitionId); - if (partition != null) { + long partitionId = tabletMeta.getPhysicalPartitionId(); + PhysicalPartition physicalPartition = table.getPhysicalPartition(partitionId); + if (physicalPartition == null) { + physicalPartition = recycleBin.getPhysicalPartition(partitionId); + if (physicalPartition != null) { isInRecycleBin = true; } else { deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId, @@ -431,7 +431,7 @@ public void checkTabletMetaConsistency(Map creatingTableIds) { // validate index long indexId = tabletMeta.getIndexId(); - MaterializedIndex index = partition.getIndex(indexId); + MaterializedIndex index = physicalPartition.getIndex(indexId); if (index == null) { deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId, "materialized index " + dbId + "." + tableId + "." + diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java index 9743b026d0cf7..e0ad98240e9f0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java @@ -41,7 +41,6 @@ public class TabletMeta { private final long dbId; private final long tableId; - private final long partitionId; private final long physicalPartitionId; private final long indexId; @@ -59,11 +58,10 @@ public class TabletMeta { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartitionId, long indexId, int schemaHash, + public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash, TStorageMedium storageMedium, boolean isLakeTablet) { this.dbId = dbId; this.tableId = tableId; - this.partitionId = partitionId; this.physicalPartitionId = physicalPartitionId; this.indexId = indexId; @@ -75,15 +73,9 @@ public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartit this.isLakeTablet = isLakeTablet; } - // for single physical partition, the physicalPartitionId is same as partitionId - public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash, - TStorageMedium storageMedium, boolean isLakeTablet) { - this(dbId, tableId, partitionId, partitionId, indexId, schemaHash, storageMedium, isLakeTablet); - } - - public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash, + public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash, TStorageMedium storageMedium) { - this(dbId, tableId, partitionId, indexId, schemaHash, storageMedium, false); + this(dbId, tableId, physicalPartitionId, indexId, schemaHash, storageMedium, false); } public long getDbId() { @@ -94,10 +86,6 @@ public long getTableId() { return tableId; } - public long getPartitionId() { - return partitionId; - } - public long getPhysicalPartitionId() { return physicalPartitionId; } @@ -164,7 +152,6 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append("dbId=").append(dbId); sb.append(" tableId=").append(tableId); - sb.append(" partitionId=").append(partitionId); sb.append(" physicalPartitionId=").append(physicalPartitionId); sb.append(" indexId=").append(indexId); sb.append(" oldSchemaHash=").append(oldSchemaHash); diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java index 8eff0d41ea271..a76426a554f1f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java @@ -820,7 +820,7 @@ private ColocateMatchResult doMatchOneGroup(GroupId groupId, TabletSchedCtx.Type.REPAIR, // physical partition id is same as partition id // since colocate table should have only one physical partition - db.getId(), tableId, partition.getId(), partition.getId(), + db.getId(), tableId, partition.getId(), index.getId(), tablet.getId(), System.currentTimeMillis()); // the tablet status will be checked and set again when being scheduled diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java index 680a8b3a15de1..580c02b719a32 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java @@ -544,9 +544,13 @@ private List balanceClusterDisk(ClusterLoadStatistic clusterStat if (olapTable == null) { continue; } + PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { + continue; + } if (isDestBackendLocationMismatch(olapTable, hBackend.getId(), lBackend.getId(), - tabletMeta.getPartitionId(), tabletId)) { + physicalPartition.getParentId(), tabletId)) { continue; } @@ -584,7 +588,7 @@ private List balanceClusterDisk(ClusterLoadStatistic clusterStat hState.minusUsedCapacity(replica.getPathHash(), replica.getDataSize()); TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, - tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(), + tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, System.currentTimeMillis()); schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW); @@ -800,7 +804,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent, // NOTICE: state has been changed, the tablet must be selected destPathUsedCap += replica.getDataSize(); srcPathUsedCap -= replica.getDataSize(); - Pair p = Pair.create(tabletMeta.getPartitionId(), tabletMeta.getIndexId()); + Pair p = Pair.create(tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId()); // p: partition // k: partition same to p srcPathPartitionTablets.compute(p, (k, pTablets) -> { @@ -819,7 +823,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent, TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, tabletMeta.getDbId(), - tabletMeta.getTableId(), tabletMeta.getPartitionId(), + tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, System.currentTimeMillis()); schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW); @@ -1414,7 +1418,7 @@ private TabletSchedCtx tryToBalanceTablet(Pair> srcTablets, } TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, - tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(), + tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, System.currentTimeMillis()); schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW); @@ -1536,14 +1540,21 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, } try { +<<<<<<< HEAD db.readLock(); PhysicalPartition partition = globalStateMgr.getPhysicalPartitionIncludeRecycleBin( olapTable, tabletMeta.getPhysicalPartitionId()); if (partition == null) { +======= + locker.lockDatabase(db, LockType.READ); + PhysicalPartition physicalPartition = globalStateMgr.getLocalMetastore() + .getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) return true; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId()); if (index == null) { return true; } @@ -1553,8 +1564,13 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, return true; } +<<<<<<< HEAD short replicaNum = globalStateMgr .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); +======= + short replicaNum = globalStateMgr.getLocalMetastore() + .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), physicalPartition.getParentId()); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicaNum == (short) -1) { return true; } @@ -1562,8 +1578,13 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, Pair statusPair = TabletChecker.getTabletHealthStatusWithPriority( tablet, +<<<<<<< HEAD GlobalStateMgr.getCurrentSystemInfo(), partition.getVisibleVersion(), +======= + globalStateMgr.getNodeMgr().getClusterInfo(), + physicalPartition.getVisibleVersion(), +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) replicaNum, aliveBeIds, olapTable.getLocation()); diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java index f5f1788a7d657..3bb74d8629816 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java @@ -299,10 +299,15 @@ private void doCheck(boolean isUrgent) { } OlapTable olapTbl = (OlapTable) table; +<<<<<<< HEAD for (Partition partition : GlobalStateMgr.getCurrentState().getAllPartitionsIncludeRecycleBin(olapTbl)) { +======= + for (PhysicalPartition physicalPartition : GlobalStateMgr.getCurrentState().getLocalMetastore() + .getAllPhysicalPartitionsIncludeRecycleBin(olapTbl)) { +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) partitionChecked++; - boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), partition.getId()); + boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), physicalPartition.getId()); totStat.isUrgentPartitionHealthy = true; if ((isUrgent && !isPartitionUrgent) || (!isUrgent && isPartitionUrgent)) { continue; @@ -322,34 +327,50 @@ private void doCheck(boolean isUrgent) { if (GlobalStateMgr.getCurrentState().getTableIncludeRecycleBin(db, olapTbl.getId()) == null) { continue TABLE; } +<<<<<<< HEAD if (GlobalStateMgr.getCurrentState() .getPartitionIncludeRecycleBin(olapTbl, partition.getId()) == null) { +======= + if (GlobalStateMgr.getCurrentState().getLocalMetastore() + .getPhysicalPartitionIncludeRecycleBin(olapTbl, physicalPartition.getId()) == null) { +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) continue; } } - if (partition.getState() != PartitionState.NORMAL) { + Partition logicalPartition = olapTbl.getPartition(physicalPartition.getParentId()); + if (logicalPartition == null) { + continue; + } + + if (logicalPartition.getState() != PartitionState.NORMAL) { // when alter job is in FINISHING state, partition state will be set to NORMAL, // and we can schedule the tablets in it. continue; } short replicaNum = GlobalStateMgr.getCurrentState() +<<<<<<< HEAD .getReplicationNumIncludeRecycleBin(olapTbl.getPartitionInfo(), partition.getId()); +======= + .getLocalMetastore() + .getReplicationNumIncludeRecycleBin( + olapTbl.getPartitionInfo(), physicalPartition.getParentId()); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicaNum == (short) -1) { continue; } - TabletCheckerStat partitionTabletCheckerStat = doCheckOnePartition(db, olapTbl, partition, + TabletCheckerStat partitionTabletCheckerStat = doCheckOnePartition(db, olapTbl, physicalPartition, replicaNum, aliveBeIdsInCluster, isPartitionUrgent); totStat.accumulateStat(partitionTabletCheckerStat); if (totStat.isUrgentPartitionHealthy && isPartitionUrgent) { // if all replicas in this partition are healthy, remove this partition from // priorities. LOG.debug("partition is healthy, remove from urgent table: {}-{}-{}", - db.getId(), olapTbl.getId(), partition.getId()); + db.getId(), olapTbl.getId(), physicalPartition.getId()); removeFromUrgentTable(new RepairTabletInfo(db.getId(), - olapTbl.getId(), Lists.newArrayList(partition.getId()))); + olapTbl.getId(), Lists.newArrayList(physicalPartition.getId()))); } } // partitions } // tables @@ -394,12 +415,12 @@ public void accumulateStat(TabletCheckerStat stat) { } } - private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, Partition partition, + private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, PhysicalPartition physicalPartition, int replicaNum, List aliveBeIdsInCluster, boolean isPartitionUrgent) { TabletCheckerStat partitionTabletCheckerStat = new TabletCheckerStat(); // Tablet in SHADOW index can not be repaired or balanced - for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + if (physicalPartition != null) { for (MaterializedIndex idx : physicalPartition.getMaterializedIndices( IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { @@ -444,7 +465,7 @@ private TabletCheckerStat doCheckOnePartition(Database db, OlapTable olapTbl, Pa TabletSchedCtx tabletSchedCtx = new TabletSchedCtx( TabletSchedCtx.Type.REPAIR, - db.getId(), olapTbl.getId(), partition.getId(), + db.getId(), olapTbl.getId(), physicalPartition.getId(), idx.getId(), tablet.getId(), System.currentTimeMillis()); // the tablet status will be set again when being scheduled @@ -516,7 +537,7 @@ private void cleanInvalidUrgentTable() { } Set parts = tblEntry.getValue(); - parts = parts.stream().filter(p -> (tbl.getPartition(p.partId) != null && !p.isTimeout())).collect( + parts = parts.stream().filter(p -> (tbl.getPhysicalPartition(p.partId) != null && !p.isTimeout())).collect( Collectors.toSet()); if (parts.isEmpty()) { deletedUrgentTable.add(Pair.create(dbId, tblId)); @@ -619,14 +640,15 @@ public static RepairTabletInfo getRepairTabletInfo(String dbName, String tblName OlapTable olapTable = (OlapTable) tbl; if (partitions == null || partitions.isEmpty()) { - partIds = olapTable.getPartitions().stream().map(Partition::getId).collect(Collectors.toList()); + partIds = olapTable.getPhysicalPartitions().stream().map(PhysicalPartition::getId).collect(Collectors.toList()); } else { for (String partName : partitions) { Partition partition = olapTable.getPartition(partName); if (partition == null) { throw new DdlException("Partition does not exist: " + partName); } - partIds.add(partition.getId()); + partIds.addAll(partition.getSubPartitions().stream() + .map(PhysicalPartition::getId).collect(Collectors.toList())); } } } finally { diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java index e651e6733858c..317904ea3eff5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java @@ -177,7 +177,6 @@ public enum State { private final long dbId; private final long tblId; - private final long partitionId; private final long physicalPartitionId; private final long indexId; private final long tabletId; @@ -229,12 +228,11 @@ public enum State { */ private int replicaNum; - public TabletSchedCtx(Type type, long dbId, long tblId, long partId, long physicalPartitionId, + public TabletSchedCtx(Type type, long dbId, long tblId, long physicalPartitionId, long idxId, long tabletId, long createTime) { this.type = type; this.dbId = dbId; this.tblId = tblId; - this.partitionId = partId; this.physicalPartitionId = physicalPartitionId; this.indexId = idxId; this.tabletId = tabletId; @@ -244,6 +242,7 @@ public TabletSchedCtx(Type type, long dbId, long tblId, long partId, long physic } @VisibleForTesting +<<<<<<< HEAD public TabletSchedCtx(Type type, long dbId, long tblId, long partId, long idxId, long tabletId, long createTime) { this(type, dbId, tblId, partId, partId, idxId, tabletId, createTime); @@ -251,12 +250,14 @@ public TabletSchedCtx(Type type, long dbId, long tblId, long partId, @VisibleForTesting public TabletSchedCtx(Type type, long dbId, long tblId, long partId, +======= + public TabletSchedCtx(Type type, long dbId, long tblId, long physicalPartitionId, +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) long idxId, long tabletId, long createTime, SystemInfoService infoService) { this.type = type; this.dbId = dbId; this.tblId = tblId; - this.partitionId = partId; - this.physicalPartitionId = partId; + this.physicalPartitionId = physicalPartitionId; this.indexId = idxId; this.tabletId = tabletId; this.createTime = createTime; @@ -333,10 +334,6 @@ public long getTblId() { return tblId; } - public long getPartitionId() { - return partitionId; - } - public long getPhysicalPartitionId() { return physicalPartitionId; } @@ -932,6 +929,10 @@ public CreateReplicaTask createEmptyReplicaAndTask() throws SchedException { if (olapTable == null) { throw new SchedException(Status.UNRECOVERABLE, "table " + tblId + " does not exist"); } + PhysicalPartition physicalPartition = olapTable.getPhysicalPartition(physicalPartitionId); + if (physicalPartition == null) { + throw new SchedException(Status.UNRECOVERABLE, "physical partition " + physicalPartitionId + " does not exist"); + } MaterializedIndexMeta indexMeta = olapTable.getIndexMetaByIndexId(indexId); if (indexMeta == null) { throw new SchedException(Status.UNRECOVERABLE, "materialized view " + indexId + " does not exist"); @@ -962,7 +963,7 @@ public CreateReplicaTask createEmptyReplicaAndTask() throws SchedException { .setStorageMedium(TStorageMedium.HDD) .setEnablePersistentIndex(olapTable.enablePersistentIndex()) .setPrimaryIndexCacheExpireSec(olapTable.primaryIndexCacheExpireSec()) - .setTabletType(olapTable.getPartitionInfo().getTabletType(partitionId)) + .setTabletType(olapTable.getPartitionInfo().getTabletType(physicalPartition.getParentId())) .setCompressionType(olapTable.getCompressionType()) .setRecoverySource(RecoverySource.SCHEDULER) .setTabletSchema(tabletSchema) @@ -1048,7 +1049,12 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) } short replicationNum = +<<<<<<< HEAD globalStateMgr.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partitionId); +======= + globalStateMgr.getLocalMetastore() + .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicationNum == (short) -1) { throw new SchedException(Status.UNRECOVERABLE, "invalid replication number"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java index 1a66ee6678f2e..3e00589272264 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java @@ -409,7 +409,7 @@ public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long tblId, PriorityQueue newPendingTablets = new PriorityQueue<>(); for (TabletSchedCtx tabletCtx : pendingTablets) { if (tabletCtx.getDbId() == dbId && tabletCtx.getTblId() == tblId - && partitionIds.contains(tabletCtx.getPartitionId())) { + && partitionIds.contains(tabletCtx.getPhysicalPartitionId())) { tabletCtx.setOrigPriority(Priority.VERY_HIGH); } newPendingTablets.add(tabletCtx); @@ -549,9 +549,10 @@ protected boolean checkIfTabletExpired(TabletSchedCtx ctx, CatalogRecycleBin rec LOG.warn("discard ctx because table {} will erase soon: {}", tableId, ctx); return true; } - long partitionId = ctx.getPartitionId(); - if (recycleBin.getPartition(partitionId) != null - && !recycleBin.ensureEraseLater(partitionId, currentTimeMs)) { + long partitionId = ctx.getPhysicalPartitionId(); + PhysicalPartition physicalPartition = recycleBin.getPhysicalPartition(partitionId); + if (physicalPartition != null + && !recycleBin.ensureEraseLater(physicalPartition.getParentId(), currentTimeMs)) { LOG.warn("discard ctx because partition {} will erase soon: {}", partitionId, ctx); return true; } @@ -707,6 +708,7 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) OlapTableState tableState = tbl.getState(); +<<<<<<< HEAD Partition partition = GlobalStateMgr.getCurrentState() .getPartitionIncludeRecycleBin(tbl, tabletCtx.getPartitionId()); if (partition == null) { @@ -715,22 +717,39 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) short replicaNum = GlobalStateMgr.getCurrentState() .getReplicationNumIncludeRecycleBin(tbl.getPartitionInfo(), partition.getId()); +======= + PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState() + .getLocalMetastore().getPhysicalPartitionIncludeRecycleBin(tbl, tabletCtx.getPhysicalPartitionId()); + if (physicalPartition == null) { + throw new SchedException(Status.UNRECOVERABLE, "physical partition " + + tabletCtx.getPhysicalPartitionId() + "does not exist"); + } + + Partition logicalPartition = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getPartitionIncludeRecycleBin(tbl, physicalPartition.getParentId()); + if (logicalPartition == null) { + throw new SchedException(Status.UNRECOVERABLE, "partition " + + physicalPartition.getParentId() + "does not exist"); + } + + short replicaNum = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getReplicationNumIncludeRecycleBin(tbl.getPartitionInfo(), physicalPartition.getParentId()); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicaNum == (short) -1) { throw new SchedException(Status.UNRECOVERABLE, "invalid replication number"); } +<<<<<<< HEAD DataProperty dataProperty = GlobalStateMgr.getCurrentState() .getDataPropertyIncludeRecycleBin(tbl.getPartitionInfo(), partition.getId()); +======= + DataProperty dataProperty = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getDataPropertyIncludeRecycleBin(tbl.getPartitionInfo(), physicalPartition.getParentId()); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (dataProperty == null) { throw new SchedException(Status.UNRECOVERABLE, "partition data property not exist"); } - PhysicalPartition physicalPartition = partition.getSubPartition(tabletCtx.getPhysicalPartitionId()); - if (physicalPartition == null) { - throw new SchedException(Status.UNRECOVERABLE, "physical partition " - + tabletCtx.getPhysicalPartitionId() + "does not exist"); - } - MaterializedIndex idx = physicalPartition.getIndex(tabletCtx.getIndexId()); if (idx == null) { throw new SchedException(Status.UNRECOVERABLE, "index does not exist"); @@ -780,7 +799,7 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) } if (statusPair.first != TabletHealthStatus.VERSION_INCOMPLETE - && (partition.getState() != PartitionState.NORMAL || tableState != OlapTableState.NORMAL) + && (logicalPartition.getState() != PartitionState.NORMAL || tableState != OlapTableState.NORMAL) && tableState != OlapTableState.WAITING_STABLE) { // If table is under ALTER process(before FINISHING), do not allow to add or delete replica. // VERSION_INCOMPLETE will repair the replica in place, which is allowed. @@ -2047,7 +2066,7 @@ public TGetTabletScheduleResponse getTabletSchedule(TGetTabletScheduleRequest re if (tabletId != -1) { all = all.filter(t -> t.getTabletId() == tabletId); } else if (partitionId != -1) { - all = all.filter(t -> t.getPartitionId() == partitionId); + all = all.filter(t -> t.getPhysicalPartitionId() == partitionId); } else if (tableId != -1) { all = all.filter(t -> t.getTblId() == tableId); } @@ -2074,15 +2093,19 @@ private void checkMetaExist(TabletSchedCtx ctx) throws SchedException { throw new SchedException(Status.UNRECOVERABLE, "table " + ctx.getTblId() + " dose not exist"); } +<<<<<<< HEAD Partition partition = GlobalStateMgr.getCurrentState().getPartitionIncludeRecycleBin(tbl, ctx.getPartitionId()); if (partition == null) { throw new SchedException(Status.UNRECOVERABLE, "partition " + ctx.getPartitionId() + " dose not exist"); } PhysicalPartition physicalPartition = partition.getSubPartition(ctx.getPhysicalPartitionId()); +======= + PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState().getLocalMetastore() + .getPhysicalPartitionIncludeRecycleBin(tbl, ctx.getPhysicalPartitionId()); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (physicalPartition == null) { - throw new SchedException(Status.UNRECOVERABLE, - "physical partition " + ctx.getPhysicalPartitionId() + " dose not exist"); + throw new SchedException(Status.UNRECOVERABLE, "partition " + ctx.getPhysicalPartitionId() + " dose not exist"); } MaterializedIndex idx = physicalPartition.getIndex(ctx.getIndexId()); diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java index bb913bb0715ca..67a4c86cd7afd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/StatisticProcDir.java @@ -149,8 +149,8 @@ public ProcResult fetchResult() throws AnalysisException { for (Partition partition : olapTable.getAllPartitions()) { short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); ++dbPartitionNum; - for (PhysicalPartition physicalParition : partition.getSubPartitions()) { - for (MaterializedIndex materializedIndex : physicalParition + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex materializedIndex : physicalPartition .getMaterializedIndices(IndexExtState.VISIBLE)) { ++dbIndexNum; for (Tablet tablet : materializedIndex.getTablets()) { @@ -167,7 +167,7 @@ public ProcResult fetchResult() throws AnalysisException { } Pair res = TabletChecker.getTabletHealthStatusWithPriority( - localTablet, infoService, physicalParition.getVisibleVersion(), + localTablet, infoService, physicalPartition.getVisibleVersion(), replicationNum, aliveBeIdsInCluster, olapTable.getLocation()); // here we treat REDUNDANT as HEALTHY, for user-friendly. diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java index 6e6a564006bcc..a492638f90cd2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java @@ -39,8 +39,9 @@ import com.starrocks.catalog.Database; import com.starrocks.catalog.LocalTablet; import com.starrocks.catalog.MaterializedIndex; +import com.starrocks.catalog.MetaObject; import com.starrocks.catalog.OlapTable; -import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Replica; import com.starrocks.catalog.Replica.ReplicaState; import com.starrocks.catalog.Table; @@ -142,20 +143,20 @@ public boolean sendTasks() { } OlapTable olapTable = (OlapTable) table; - Partition partition = olapTable.getPartition(tabletMeta.getPartitionId()); - if (partition == null) { - LOG.debug("partition[{}] does not exist", tabletMeta.getPartitionId()); + PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { + LOG.debug("partition[{}] does not exist", tabletMeta.getPhysicalPartitionId()); return false; } // check partition's replication num. if 1 replication. skip - short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); + short replicationNum = olapTable.getPartitionInfo().getReplicationNum(physicalPartition.getParentId()); if (replicationNum == (short) 1) { - LOG.debug("partition[{}]'s replication num is 1. skip consistency check", partition.getId()); + LOG.debug("partition[{}]'s replication num is 1. skip consistency check", physicalPartition.getParentId()); return false; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId()); if (index == null) { LOG.debug("index[{}] does not exist", tabletMeta.getIndexId()); return false; @@ -167,7 +168,7 @@ public boolean sendTasks() { return false; } - checkedVersion = partition.getVisibleVersion(); + checkedVersion = physicalPartition.getVisibleVersion(); checkedSchemaHash = olapTable.getSchemaHashByIndexId(tabletMeta.getIndexId()); int sentTaskReplicaNum = 0; @@ -184,12 +185,21 @@ public boolean sendTasks() { } CheckConsistencyTask task = new CheckConsistencyTask(null, replica.getBackendId(), +<<<<<<< HEAD tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(), tabletMeta.getIndexId(), tabletId, checkedSchemaHash, checkedVersion); +======= + tabletMeta.getDbId(), + tabletMeta.getTableId(), + tabletMeta.getPhysicalPartitionId(), + tabletMeta.getIndexId(), + tabletId, checkedSchemaHash, + checkedVersion); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) // add task to send batchTask.addTask(task); @@ -271,13 +281,13 @@ public synchronized int tryFinishJob() { } OlapTable olapTable = (OlapTable) table; - Partition partition = olapTable.getPartition(tabletMeta.getPartitionId()); - if (partition == null) { - LOG.warn("partition[{}] does not exist", tabletMeta.getPartitionId()); + PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId()); + if (physicalPartition == null) { + LOG.warn("partition[{}] does not exist", tabletMeta.getPhysicalPartitionId()); return -1; } - MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId()); if (index == null) { LOG.warn("index[{}] does not exist", tabletMeta.getIndexId()); return -1; @@ -355,7 +365,9 @@ public synchronized int tryFinishJob() { long lastCheckTime = System.currentTimeMillis(); db.setLastCheckTime(lastCheckTime); olapTable.setLastCheckTime(lastCheckTime); - partition.setLastCheckTime(lastCheckTime); + if (physicalPartition instanceof MetaObject) { + ((MetaObject) physicalPartition).setLastCheckTime(lastCheckTime); + } index.setLastCheckTime(lastCheckTime); tablet.setLastCheckTime(lastCheckTime); tablet.setIsConsistent(isConsistent); @@ -364,9 +376,15 @@ public synchronized int tryFinishJob() { tablet.setCheckedVersion(checkedVersion); // log +<<<<<<< HEAD ConsistencyCheckInfo info = new ConsistencyCheckInfo(db.getId(), table.getId(), partition.getId(), index.getId(), tabletId, lastCheckTime, checkedVersion, isConsistent); +======= + ConsistencyCheckInfo info = new ConsistencyCheckInfo(db.getId(), table.getId(), physicalPartition.getId(), + index.getId(), tabletId, lastCheckTime, + checkedVersion, isConsistent); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) journalTask = GlobalStateMgr.getCurrentState().getEditLog().logFinishConsistencyCheckNoWait(info); } finally { db.writeUnlock(); diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java index 8580377e4e894..1b8a2d39f650e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java @@ -315,33 +315,48 @@ protected List chooseTablets() { // sort partitions Queue partitionQueue = +<<<<<<< HEAD new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); for (PhysicalPartition partition : table.getPhysicalPartitions()) { +======= + new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); + for (PhysicalPartition physicalPartition : table.getPhysicalPartitions()) { +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) // check partition's replication num. if 1 replication. skip - if (table.getPartitionInfo().getReplicationNum(partition.getParentId()) == (short) 1) { - LOG.debug("partition[{}]'s replication num is 1. ignore", partition.getParentId()); + if (table.getPartitionInfo().getReplicationNum(physicalPartition.getParentId()) == (short) 1) { + LOG.debug("partition[{}]'s replication num is 1. ignore", physicalPartition.getParentId()); continue; } // check if this partition has no data +<<<<<<< HEAD if (partition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { LOG.debug("partition[{}]'s version is {}. ignore", partition.getId(), Partition.PARTITION_INIT_VERSION); +======= + if (physicalPartition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { + LOG.debug("partition[{}]'s version is {}. ignore", physicalPartition.getId(), + Partition.PARTITION_INIT_VERSION); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) continue; } - if (partition instanceof Partition) { - partitionQueue.add((Partition) partition); - } else if (partition instanceof PhysicalPartitionImpl) { - partitionQueue.add((PhysicalPartitionImpl) partition); + if (physicalPartition instanceof Partition) { + partitionQueue.add((Partition) physicalPartition); + } else if (physicalPartition instanceof PhysicalPartitionImpl) { + partitionQueue.add((PhysicalPartitionImpl) physicalPartition); } } while ((chosenOne = partitionQueue.poll()) != null) { - PhysicalPartition partition = (PhysicalPartition) chosenOne; + PhysicalPartition physicalPartition = (PhysicalPartition) chosenOne; // sort materializedIndices List visibleIndexes = +<<<<<<< HEAD partition.getMaterializedIndices(IndexExtState.VISIBLE); +======= + physicalPartition.getMaterializedIndices(IndexExtState.VISIBLE); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) Queue indexQueue = new PriorityQueue<>(Math.max(visibleIndexes.size(), 1), COMPARATOR); indexQueue.addAll(visibleIndexes); @@ -363,14 +378,23 @@ protected List chooseTablets() { } // check if version has already been checked - if (partition.getVisibleVersion() == tablet.getCheckedVersion()) { + if (physicalPartition.getVisibleVersion() == tablet.getCheckedVersion()) { if (tablet.isConsistent()) { LOG.debug("tablet[{}]'s version[{}-{}] has been checked. ignore", +<<<<<<< HEAD chosenTabletId, tablet.getCheckedVersion(), partition.getVisibleVersion()); } } else { LOG.info("chose tablet[{}-{}-{}-{}-{}] to check consistency", db.getId(), table.getId(), partition.getId(), index.getId(), chosenTabletId); +======= + chosenTabletId, tablet.getCheckedVersion(), + physicalPartition.getVisibleVersion()); + } + } else { + LOG.info("chose tablet[{}-{}-{}-{}-{}] to check consistency", db.getId(), + table.getId(), physicalPartition.getId(), index.getId(), chosenTabletId); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) chosenTablets.add(chosenTabletId); } @@ -416,6 +440,7 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM LOG.warn("replay finish consistency check failed, db is null, info: {}", info); return; } +<<<<<<< HEAD db.writeLock(); try { OlapTable table = (OlapTable) db.getTable(info.getTableId()); @@ -425,10 +450,22 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM } Partition partition = table.getPartition(info.getPartitionId()); if (partition == null) { +======= + OlapTable table = (OlapTable) db.getTable(info.getTableId()); + if (table == null) { + LOG.warn("replay finish consistency check failed, table is null, info: {}", info); + return; + } + + try (AutoCloseableLock ignore + = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(table.getId()), LockType.WRITE)) { + PhysicalPartition physicalPartition = table.getPhysicalPartition(info.getPhysicalPartitionId()); + if (physicalPartition == null) { +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) LOG.warn("replay finish consistency check failed, partition is null, info: {}", info); return; } - MaterializedIndex index = partition.getIndex(info.getIndexId()); + MaterializedIndex index = physicalPartition.getIndex(info.getIndexId()); if (index == null) { LOG.warn("replay finish consistency check failed, index is null, info: {}", info); return; @@ -442,7 +479,9 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM long lastCheckTime = info.getLastCheckTime(); db.setLastCheckTime(lastCheckTime); table.setLastCheckTime(lastCheckTime); - partition.setLastCheckTime(lastCheckTime); + if (physicalPartition instanceof MetaObject) { + ((MetaObject) physicalPartition).setLastCheckTime(lastCheckTime); + } index.setLastCheckTime(lastCheckTime); tablet.setLastCheckTime(lastCheckTime); tablet.setCheckedVersion(info.getCheckedVersion()); diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java b/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java index 7ae6664589a4e..0b448bf10c8f0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java @@ -86,8 +86,13 @@ public static Map> groupTabletID(Collection partitio throws NoAliveBackendException { Map> groupMap = new HashMap<>(); for (Partition partition : partitions) { +<<<<<<< HEAD for (PhysicalPartition physicalParition : partition.getSubPartitions()) { for (MaterializedIndex index : partition.getMaterializedIndices(indexState)) { +======= + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex index : physicalPartition.getMaterializedIndices(indexState)) { +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) for (Tablet tablet : index.getTablets()) { Long beId = chooseBackend((LakeTablet) tablet); if (beId == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java index aa4ef076b2cd7..2eef61805ece1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java @@ -1110,7 +1110,7 @@ private static void initTabletMeta(Tablet tablet, TTabletMeta tTabletMeta) { TabletMeta tabletMeta = GlobalStateMgr.getCurrentInvertedIndex().getTabletMeta(tablet.getId()); tTabletMeta.setDb_id(tabletMeta.getDbId()); tTabletMeta.setTable_id(tabletMeta.getTableId()); - tTabletMeta.setPartition_id(tabletMeta.getPartitionId()); + tTabletMeta.setPartition_id(tabletMeta.getPhysicalPartitionId()); tTabletMeta.setIndex_id(tabletMeta.getIndexId()); tTabletMeta.setStorage_medium(tabletMeta.getStorageMedium()); tTabletMeta.setOld_schema_hash(tabletMeta.getOldSchemaHash()); diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java index 2245e83ce2221..80dae224f1624 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java @@ -812,11 +812,6 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta continue; } - if (globalStateMgr.getLocalMetastore() - .getPartitionIncludeRecycleBin(olapTable, tabletMeta.getPartitionId()) == null) { - continue; - } - PhysicalPartition partition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, partitionId); if (partition == null) { @@ -825,7 +820,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta short replicationNum = globalStateMgr.getLocalMetastore().getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), - tabletMeta.getPartitionId()); + partition.getParentId()); if (replicationNum == (short) -1) { continue; } @@ -1363,7 +1358,7 @@ private static void handleSetTabletInMemory(long backendId, Map b long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; long partitionId = - tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; + tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; Database db = GlobalStateMgr.getCurrentState().getDb(dbId); if (db == null) { @@ -1371,15 +1366,19 @@ private static void handleSetTabletInMemory(long backendId, Map b } db.readLock(); try { +<<<<<<< HEAD OlapTable olapTable = (OlapTable) db.getTable(tableId); if (olapTable == null) { continue; } Partition partition = olapTable.getPartition(partitionId); +======= + PhysicalPartition partition = olapTable.getPhysicalPartition(partitionId); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (partition == null) { continue; } - boolean feIsInMemory = olapTable.getPartitionInfo().getIsInMemory(partitionId); + boolean feIsInMemory = olapTable.getPartitionInfo().getIsInMemory(partition.getParentId()); if (beIsInMemory != feIsInMemory) { tabletToInMemory.add(new Pair<>(tabletId, feIsInMemory)); } @@ -1722,7 +1721,6 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId); long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; - long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; long physicalPartitionId = tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; @@ -1738,6 +1736,7 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon } db.writeLock(); try { +<<<<<<< HEAD OlapTable olapTable = (OlapTable) globalStateMgr.getTableIncludeRecycleBin(db, tableId); if (olapTable == null) { throw new MetaNotFoundException("table[" + tableId + "] does not exist"); @@ -1753,12 +1752,24 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon throw new MetaNotFoundException("invalid replication number of partition [" + partitionId + "]"); } +======= +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) PhysicalPartition partition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, physicalPartitionId); if (partition == null) { throw new MetaNotFoundException("physical partition[" + physicalPartitionId + "] does not exist"); } + if (globalStateMgr.getLocalMetastore().getPartitionIncludeRecycleBin(olapTable, partition.getParentId()) == null) { + throw new MetaNotFoundException("partition[" + partition.getParentId() + "] does not exist"); + } + short replicationNum = + globalStateMgr.getLocalMetastore() + .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); + if (replicationNum == (short) -1) { + throw new MetaNotFoundException("invalid replication number of partition [" + partition.getParentId() + "]"); + } + MaterializedIndex materializedIndex = partition.getIndex(indexId); if (materializedIndex == null) { throw new MetaNotFoundException("index[" + indexId + "] does not exist"); diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java b/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java index 1c2b3a0b20d4a..0b01f9c62790c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/ConsistencyCheckInfo.java @@ -49,7 +49,7 @@ public class ConsistencyCheckInfo implements Writable { @SerializedName("tb") private long tableId; @SerializedName("pt") - private long partitionId; + private long physicalPartitionId; @SerializedName("idx") private long indexId; @SerializedName("tt") @@ -65,12 +65,12 @@ public ConsistencyCheckInfo() { // for persist } - public ConsistencyCheckInfo(long dbId, long tableId, long partitionId, long indexId, long tabletId, + public ConsistencyCheckInfo(long dbId, long tableId, long physicalPartitionId, long indexId, long tabletId, long lastCheckTime, long checkedVersion, boolean isConsistent) { this.dbId = dbId; this.tableId = tableId; - this.partitionId = partitionId; + this.physicalPartitionId = physicalPartitionId; this.indexId = indexId; this.tabletId = tabletId; @@ -88,8 +88,8 @@ public long getTableId() { return tableId; } - public long getPartitionId() { - return partitionId; + public long getPhysicalPartitionId() { + return physicalPartitionId; } public long getIndexId() { @@ -116,7 +116,7 @@ public boolean isConsistent() { public void write(DataOutput out) throws IOException { out.writeLong(dbId); out.writeLong(tableId); - out.writeLong(partitionId); + out.writeLong(physicalPartitionId); out.writeLong(indexId); out.writeLong(tabletId); @@ -130,7 +130,7 @@ public void write(DataOutput out) throws IOException { public void readFields(DataInput in) throws IOException { dbId = in.readLong(); tableId = in.readLong(); - partitionId = in.readLong(); + physicalPartitionId = in.readLong(); indexId = in.readLong(); tabletId = in.readLong(); diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index a17bb39ca4b5c..5d1bcbad81278 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -504,6 +504,533 @@ private void handleShowMaterializedView() throws AnalysisException { Authorizer.checkAnyActionOnMaterializedView(connectContext.getCurrentUserIdentity(), connectContext.getCurrentRoleIds(), new TableName(db.getFullName(), mvTable.getName())); } catch (AccessDeniedException e) { +<<<<<<< HEAD +======= + AccessDeniedException.reportAccessDenied(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, + context.getCurrentUserIdentity(), + context.getCurrentRoleIds(), + PrivilegeType.ANY.name(), ObjectType.TABLE.name(), tableName); + } + + Table table = MetaUtils.getSessionAwareTable(context, db, new TableName(dbName, tableName)); + if (table == null) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); + } + + if (!table.isNativeTableOrMaterializedView()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE, tableName); + } + + OlapTable olapTable = (OlapTable) table; + int i = 0; + long totalSize = 0; + long totalReplicaCount = 0; + + // sort by index name + Map indexNames = olapTable.getIndexNameToId(); + Map sortedIndexNames = new TreeMap<>(indexNames); + + for (Long indexId : sortedIndexNames.values()) { + long indexSize = 0; + long indexReplicaCount = 0; + long indexRowCount = 0; + for (PhysicalPartition partition : olapTable.getAllPhysicalPartitions()) { + MaterializedIndex mIndex = partition.getIndex(indexId); + indexSize += mIndex.getDataSize(); + indexReplicaCount += mIndex.getReplicaCount(); + indexRowCount += mIndex.getRowCount(); + } + + Pair indexSizePair = DebugUtil.getByteUint(indexSize); + String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(indexSizePair.first) + " " + + indexSizePair.second; + + List row = null; + if (i == 0) { + row = Arrays.asList(tableName, + olapTable.getIndexNameById(indexId), + readableSize, String.valueOf(indexReplicaCount), + String.valueOf(indexRowCount)); + } else { + row = Arrays.asList("", + olapTable.getIndexNameById(indexId), + readableSize, String.valueOf(indexReplicaCount), + String.valueOf(indexRowCount)); + } + + totalSize += indexSize; + totalReplicaCount += indexReplicaCount; + totalRows.add(row); + + i++; + } // end for indices + + Pair totalSizePair = DebugUtil.getByteUint(totalSize); + String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalSizePair.first) + " " + + totalSizePair.second; + List row = Arrays.asList("", "Total", readableSize, String.valueOf(totalReplicaCount), ""); + totalRows.add(row); + } + } catch (AnalysisException e) { + throw new SemanticException(e.getMessage()); + } finally { + locker.unLockDatabase(db, LockType.READ); + } + return new ShowResultSet(statement.getMetaData(), statement.getResultRows()); + } + + @Override + public ShowResultSet visitShowCollationStatement(ShowCollationStmt statement, ConnectContext context) { + List> rows = Lists.newArrayList(); + List row = Lists.newArrayList(); + // | utf8_general_ci | utf8 | 33 | Yes | Yes | 1 | + row.add("utf8_general_ci"); + row.add("utf8"); + row.add("33"); + row.add("Yes"); + row.add("Yes"); + row.add("1"); + rows.add(row); + // | binary | binary | 63 | Yes | Yes | 1 | + row = Lists.newArrayList(); + row.add("binary"); + row.add("binary"); + row.add("63"); + row.add("Yes"); + row.add("Yes"); + row.add("1"); + rows.add(row); + // | gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | + row = Lists.newArrayList(); + row.add("gbk_chinese_ci"); + row.add("gbk"); + row.add("28"); + row.add("Yes"); + row.add("Yes"); + row.add("1"); + rows.add(row); + return new ShowResultSet(statement.getMetaData(), rows); + } + + @Override + public ShowResultSet visitShowPartitionsStatement(ShowPartitionsStmt statement, ConnectContext context) { + ProcNodeInterface procNodeI = statement.getNode(); + Preconditions.checkNotNull(procNodeI); + try { + List> rows = ((PartitionsProcDir) procNodeI).fetchResultByFilter(statement.getFilterMap(), + statement.getOrderByPairs(), statement.getLimitElement()).getRows(); + return new ShowResultSet(statement.getMetaData(), rows); + } catch (AnalysisException e) { + throw new SemanticException(e.getMessage()); + } + } + + @Override + public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectContext context) { + List> rows = Lists.newArrayList(); + + GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); + if (statement.isShowSingleTablet()) { + long tabletId = statement.getTabletId(); + TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); + TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId); + Long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; + String dbName = null; + Long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; + String tableName = null; + Long partitionId = tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; + String partitionName = null; + Long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; + String indexName = null; + Boolean isSync = true; + + // check real meta + do { + Database db = globalStateMgr.getDb(dbId); + if (db == null) { + isSync = false; + break; + } + dbName = db.getFullName(); + + Locker locker = new Locker(); + locker.lockDatabase(db, LockType.READ); + try { + Table table = db.getTable(tableId); + if (!(table instanceof OlapTable)) { + isSync = false; + break; + } + tableName = table.getName(); + Pair privResult = Authorizer.checkPrivForShowTablet(context, dbName, table); + if (!privResult.first) { + AccessDeniedException.reportAccessDenied( + InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, + context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.ANY.name(), ObjectType.TABLE.name(), null); + } + + OlapTable olapTable = (OlapTable) table; + PhysicalPartition physicalPartition = olapTable.getPhysicalPartition(partitionId); + if (physicalPartition == null) { + isSync = false; + break; + } + Partition partition = olapTable.getPartition(physicalPartition.getParentId()); + if (partition == null) { + isSync = false; + break; + } + partitionName = partition.getName(); + + MaterializedIndex index = physicalPartition.getIndex(indexId); + if (index == null) { + isSync = false; + break; + } + indexName = olapTable.getIndexNameById(indexId); + + if (table.isCloudNativeTableOrMaterializedView()) { + break; + } + + LocalTablet tablet = (LocalTablet) index.getTablet(tabletId); + if (tablet == null) { + isSync = false; + break; + } + + List replicas = tablet.getImmutableReplicas(); + for (Replica replica : replicas) { + Replica tmp = invertedIndex.getReplica(tabletId, replica.getBackendId()); + if (tmp == null) { + isSync = false; + break; + } + // use !=, not equals(), because this should be the same object. + if (tmp != replica) { + isSync = false; + break; + } + } + + } finally { + locker.unLockDatabase(db, LockType.READ); + } + } while (false); + + String detailCmd = String.format("SHOW PROC '/dbs/%d/%d/partitions/%d/%d/%d';", + dbId, tableId, partitionId, indexId, tabletId); + rows.add(Lists.newArrayList(dbName, tableName, partitionName, indexName, + dbId.toString(), tableId.toString(), + partitionId.toString(), indexId.toString(), + isSync.toString(), detailCmd)); + } else { + Database db = globalStateMgr.getDb(statement.getDbName()); + MetaUtils.checkDbNullAndReport(db, statement.getDbName()); + + Locker locker = new Locker(); + locker.lockDatabase(db, LockType.READ); + try { + Table table = MetaUtils.getSessionAwareTable( + context, db, new TableName(statement.getDbName(), statement.getTableName())); + if (table == null) { + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, statement.getTableName()); + } + if (!table.isNativeTableOrMaterializedView()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_NOT_OLAP_TABLE, statement.getTableName()); + } + + Pair privResult = Authorizer.checkPrivForShowTablet( + context, db.getFullName(), table); + if (!privResult.first) { + AccessDeniedException.reportAccessDenied( + InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, + context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + PrivilegeType.ANY.name(), ObjectType.TABLE.name(), null); + } + Boolean hideIpPort = privResult.second; + statement.setTable(table); + + OlapTable olapTable = (OlapTable) table; + long sizeLimit = -1; + if (statement.hasOffset() && statement.hasLimit()) { + sizeLimit = statement.getOffset() + statement.getLimit(); + } else if (statement.hasLimit()) { + sizeLimit = statement.getLimit(); + } + boolean stop = false; + Collection partitions = new ArrayList<>(); + if (statement.hasPartition()) { + PartitionNames partitionNames = statement.getPartitionNames(); + for (String partName : partitionNames.getPartitionNames()) { + Partition partition = olapTable.getPartition(partName, partitionNames.isTemp()); + if (partition == null) { + throw new SemanticException("Unknown partition: " + partName); + } + partitions.add(partition); + } + } else { + partitions = olapTable.getPartitions(); + } + List> tabletInfos = new ArrayList<>(); + String indexName = statement.getIndexName(); + long indexId = -1; + if (indexName != null) { + Long id = olapTable.getIndexIdByName(indexName); + if (id == null) { + // invalid indexName + ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, statement.getIndexName()); + } + indexId = id; + } + for (Partition partition : partitions) { + if (stop) { + break; + } + for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { + for (MaterializedIndex index : physicalPartition.getMaterializedIndices(IndexExtState.ALL)) { + if (indexId > -1 && index.getId() != indexId) { + continue; + } + if (olapTable.isCloudNativeTableOrMaterializedView()) { + LakeTabletsProcDir procNode = new LakeTabletsProcDir(db, olapTable, index); + tabletInfos.addAll(procNode.fetchComparableResult()); + } else { + LocalTabletsProcDir procDir = new LocalTabletsProcDir(db, olapTable, index); + tabletInfos.addAll(procDir.fetchComparableResult( + statement.getVersion(), statement.getBackendId(), statement.getReplicaState(), + hideIpPort)); + } + if (sizeLimit > -1 && CollectionUtils.isEmpty(statement.getOrderByPairs()) + && tabletInfos.size() >= sizeLimit) { + stop = true; + break; + } + } + } + } + + // order by + List orderByPairs = statement.getOrderByPairs(); + ListComparator> comparator; + if (orderByPairs != null) { + OrderByPair[] orderByPairArr = new OrderByPair[orderByPairs.size()]; + comparator = new ListComparator<>(orderByPairs.toArray(orderByPairArr)); + } else { + // order by tabletId, replicaId + comparator = new ListComparator<>(0, 1); + } + tabletInfos.sort(comparator); + + if (sizeLimit > -1 && tabletInfos.size() >= sizeLimit) { + tabletInfos = tabletInfos.subList((int) statement.getOffset(), (int) sizeLimit); + } + + for (List tabletInfo : tabletInfos) { + List oneTablet = new ArrayList<>(tabletInfo.size()); + for (Comparable column : tabletInfo) { + oneTablet.add(column.toString()); + } + rows.add(oneTablet); + } + } finally { + locker.unLockDatabase(db, LockType.READ); + } + } + + return new ShowResultSet(statement.getMetaData(), rows); + } + + @Override + public ShowResultSet visitShowBackupStatement(ShowBackupStmt statement, ConnectContext context) { + Database filterDb = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); + List> infos = Lists.newArrayList(); + List dbs = Lists.newArrayList(); + + if (filterDb == null) { + for (Map.Entry entry : GlobalStateMgr.getCurrentState() + .getLocalMetastore().getIdToDb().entrySet()) { + dbs.add(entry.getValue()); + } + } else { + dbs.add(filterDb); + } + + for (Database db : dbs) { + AbstractJob jobI = GlobalStateMgr.getCurrentState().getBackupHandler().getJob(db.getId()); + if (jobI == null || !(jobI instanceof BackupJob)) { + // show next db + continue; + } + + BackupJob backupJob = (BackupJob) jobI; + + // check privilege + List tableRefs = backupJob.getTableRef(); + AtomicBoolean privilegeDeny = new AtomicBoolean(false); + tableRefs.forEach(tableRef -> { + TableName tableName = tableRef.getName(); + try { + Authorizer.checkTableAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), + tableName.getDb(), tableName.getTbl(), PrivilegeType.EXPORT); + } catch (AccessDeniedException e) { + privilegeDeny.set(true); + } + }); + if (privilegeDeny.get()) { + return new ShowResultSet(statement.getMetaData(), EMPTY_SET); + } + + List info = backupJob.getInfo(); + infos.add(info); + } + return new ShowResultSet(statement.getMetaData(), infos); + } + + @Override + public ShowResultSet visitShowRestoreStatement(ShowRestoreStmt statement, ConnectContext context) { + Database filterDb = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); + List> infos = Lists.newArrayList(); + List dbs = Lists.newArrayList(); + + if (filterDb == null) { + for (Map.Entry entry : GlobalStateMgr.getCurrentState() + .getLocalMetastore().getIdToDb().entrySet()) { + dbs.add(entry.getValue()); + } + } else { + dbs.add(filterDb); + } + + for (Database db : dbs) { + AbstractJob jobI = GlobalStateMgr.getCurrentState().getBackupHandler().getJob(db.getId()); + if (jobI == null || !(jobI instanceof RestoreJob)) { + // show next db + continue; + } + + RestoreJob restoreJob = (RestoreJob) jobI; + List info = restoreJob.getInfo(); + infos.add(info); + } + return new ShowResultSet(statement.getMetaData(), infos); + } + + @Override + public ShowResultSet visitShowBrokerStatement(ShowBrokerStmt statement, ConnectContext context) { + List> rowSet = GlobalStateMgr.getCurrentState().getBrokerMgr().getBrokersInfo(); + + // Only success + return new ShowResultSet(statement.getMetaData(), rowSet); + } + + @Override + public ShowResultSet visitShowResourceStatement(ShowResourcesStmt statement, ConnectContext context) { + List> rowSet = GlobalStateMgr.getCurrentState().getResourceMgr().getResourcesInfo(); + + // Only success + return new ShowResultSet(statement.getMetaData(), rowSet); + } + + @Override + public ShowResultSet visitShowExportStatement(ShowExportStmt statement, ConnectContext context) { + GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); + Database db = globalStateMgr.getDb(statement.getDbName()); + MetaUtils.checkDbNullAndReport(db, statement.getDbName()); + long dbId = db.getId(); + + ExportMgr exportMgr = globalStateMgr.getExportMgr(); + + Set states = null; + ExportJob.JobState state = statement.getJobState(); + if (state != null) { + states = Sets.newHashSet(state); + } + List> infos = exportMgr.getExportJobInfosByIdOrState( + dbId, statement.getJobId(), states, statement.getQueryId(), + statement.getOrderByPairs(), statement.getLimit()); + + return new ShowResultSet(statement.getMetaData(), infos); + } + + @Override + public ShowResultSet visitShowBackendsStatement(ShowBackendsStmt statement, ConnectContext context) { + List> backendInfos = BackendsProcDir.getClusterBackendInfos(); + return new ShowResultSet(statement.getMetaData(), backendInfos); + } + + @Override + public ShowResultSet visitShowFrontendsStatement(ShowFrontendsStmt statement, ConnectContext context) { + List> infos = Lists.newArrayList(); + FrontendsProcNode.getFrontendsInfo(GlobalStateMgr.getCurrentState(), infos); + return new ShowResultSet(statement.getMetaData(), infos); + } + + @Override + public ShowResultSet visitShowRepositoriesStatement(ShowRepositoriesStmt statement, ConnectContext context) { + List> repoInfos = GlobalStateMgr.getCurrentState().getBackupHandler().getRepoMgr().getReposInfo(); + return new ShowResultSet(statement.getMetaData(), repoInfos); + } + + @Override + public ShowResultSet visitShowSnapshotStatement(ShowSnapshotStmt statement, ConnectContext context) { + Repository repo = GlobalStateMgr.getCurrentState().getBackupHandler().getRepoMgr().getRepo(statement.getRepoName()); + if (repo == null) { + throw new SemanticException("Repository " + statement.getRepoName() + " does not exist"); + } + + List> snapshotInfos = repo.getSnapshotInfos(statement.getSnapshotName(), statement.getTimestamp(), + statement.getSnapshotNames()); + return new ShowResultSet(statement.getMetaData(), snapshotInfos); + } + + @Override + public ShowResultSet visitShowGrantsStatement(ShowGrantsStmt statement, ConnectContext context) { + AuthorizationMgr authorizationManager = GlobalStateMgr.getCurrentState().getAuthorizationMgr(); + try { + List> infos = new ArrayList<>(); + if (statement.getRole() != null) { + List granteeRole = authorizationManager.getGranteeRoleDetailsForRole(statement.getRole()); + if (granteeRole != null) { + infos.add(granteeRole); + } + + Map> typeToPrivilegeEntryList = + authorizationManager.getTypeToPrivilegeEntryListByRole(statement.getRole()); + infos.addAll(privilegeToRowString(authorizationManager, + new GrantRevokeClause(null, statement.getRole()), typeToPrivilegeEntryList)); + } else { + List granteeRole = authorizationManager.getGranteeRoleDetailsForUser(statement.getUserIdent()); + if (granteeRole != null) { + infos.add(granteeRole); + } + + Map> typeToPrivilegeEntryList = + authorizationManager.getTypeToPrivilegeEntryListByUser(statement.getUserIdent()); + infos.addAll(privilegeToRowString(authorizationManager, + new GrantRevokeClause(statement.getUserIdent(), null), typeToPrivilegeEntryList)); + } + return new ShowResultSet(statement.getMetaData(), infos); + } catch (PrivilegeException e) { + throw new SemanticException(e.getMessage()); + } + } + + private List> privilegeToRowString(AuthorizationMgr authorizationManager, GrantRevokeClause userOrRoleName, + Map> typeToPrivilegeEntryList) + throws PrivilegeException { + List> infos = new ArrayList<>(); + for (Map.Entry> typeToPrivilegeEntry + : typeToPrivilegeEntryList.entrySet()) { + for (PrivilegeEntry privilegeEntry : typeToPrivilegeEntry.getValue()) { + ObjectType objectType = typeToPrivilegeEntry.getKey(); + String catalogName; + try { + catalogName = getCatalogNameFromPEntry(objectType, privilegeEntry); + } catch (MetaNotFoundException e) { + // ignore this entry +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java index bbad48c914164..bb45eee9ad619 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/mv/BinlogConsumeStateVO.java @@ -51,7 +51,7 @@ public List toThrift() { TabletMeta meta = tabletIndex.getTabletMeta(key.getTabletId()); scan.setTable_id(meta.getTableId()); scan.setTablet_id(key.getTabletId()); - scan.setPartition_id(meta.getPartitionId()); + scan.setPartition_id(meta.getPhysicalPartitionId()); scan.setOffset(value.toThrift()); res.add(scan); }); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index e3126fcad011d..adba96fca9079 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -346,7 +346,7 @@ public void recreateTabletInvertIndex() { .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partition.getParentId(), physicalPartitionId, + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, physicalPartitionId, indexId, schemaHash, medium, table.isCloudNativeTableOrMaterializedView()); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -1695,9 +1695,14 @@ private PhysicalPartition createPhysicalPartition(Database db, OlapTable olapTab shardGroupId = GlobalStateMgr.getCurrentStarOSAgent(). createShardGroup(db.getId(), olapTable.getId(), id); } +<<<<<<< HEAD PhysicalPartitionImpl physicalParition = new PhysicalPartitionImpl( id, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); +======= + PhysicalPartitionImpl physicalPartition = new PhysicalPartitionImpl( + id, name, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); +>>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) PartitionInfo partitionInfo = olapTable.getPartitionInfo(); short replicationNum = partitionInfo.getReplicationNum(partitionId); @@ -1718,15 +1723,15 @@ private PhysicalPartition createPhysicalPartition(Database db, OlapTable olapTab tabletMeta, tabletIdSet); } else { createOlapTablets(olapTable, index, Replica.ReplicaState.NORMAL, distributionInfo, - physicalParition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); + physicalPartition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); } if (index.getId() != olapTable.getBaseIndexId()) { // add rollup index to partition - physicalParition.createRollupIndex(index); + physicalPartition.createRollupIndex(index); } } - return physicalParition; + return physicalPartition; } public void addSubPartitions(Database db, String tableName, @@ -1800,7 +1805,7 @@ public void replayAddSubPartition(PhysicalPartitionPersistInfoV2 info) throws Dd for (MaterializedIndex index : physicalPartition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), info.getPartitionId(), + TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), physicalPartition.getId(), index.getId(), schemaHash, olapTable.getPartitionInfo().getDataProperty( info.getPartitionId()).getStorageMedium(), false); for (Tablet tablet : index.getTablets()) { @@ -2328,7 +2333,7 @@ public void replayCreateTable(CreateTableInfo info) { .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { long indexId = mIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partition.getParentId(), physicalPartitionId, + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, physicalPartitionId, indexId, schemaHash, medium, table.isCloudNativeTableOrMaterializedView()); for (Tablet tablet : mIndex.getTablets()) { long tabletId = tablet.getId(); @@ -2861,6 +2866,12 @@ public Collection getAllPartitionsIncludeRecycleBin(OlapTable table) return partitions; } + public Collection getAllPhysicalPartitionsIncludeRecycleBin(OlapTable table) { + Collection partitions = table.getAllPhysicalPartitions(); + partitions.addAll(recycleBin.getPhysicalPartitions(table.getId())); + return partitions; + } + // NOTE: result can be null, cause partition erase is not in db lock public DataProperty getDataPropertyIncludeRecycleBin(PartitionInfo info, long partitionId) { DataProperty dataProperty = info.getDataProperty(partitionId); @@ -4883,7 +4894,7 @@ private void setReplicaStatusInternal(long tabletId, long backendId, Replica.Rep if (!isReplay) { // Put this tablet into urgent table so that it can be repaired ASAP. stateMgr.getTabletChecker().setTabletForUrgentRepair(dbId, meta.getTableId(), - meta.getPartitionId()); + meta.getPhysicalPartitionId()); SetReplicaStatusOperationLog log = new SetReplicaStatusOperationLog(backendId, tabletId, status); GlobalStateMgr.getCurrentState().getEditLog().logSetReplicaStatus(log); diff --git a/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java b/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java index 0f42691e59f0f..95ee67f0e9c40 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/TabletMetadataUpdateAgentTaskFactory.java @@ -127,7 +127,7 @@ public List getTTabletMetaInfoList() { } TTabletMetaInfo metaInfo = new TTabletMetaInfo(); metaInfo.setTablet_id(tabletId); - metaInfo.setPartition_id(tabletMeta.getPartitionId()); + metaInfo.setPartition_id(tabletMeta.getPhysicalPartitionId()); metaInfo.setMeta_type(TTabletMetaType.PARTITIONID); metaInfos.add(metaInfo); // add at most 10000 tablet meta during one sync to avoid too large task diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java b/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java index 87e4b4597bf51..02f44c49dc696 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnStateListener.java @@ -102,11 +102,11 @@ public void preCommit(TransactionState txnState, List finished if (tabletMeta.getTableId() != table.getId()) { continue; } - if (table.getPhysicalPartition(tabletMeta.getPartitionId()) == null) { + if (table.getPhysicalPartition(tabletMeta.getPhysicalPartitionId()) == null) { // this can happen when partitionId == -1 (tablet being dropping) or partition really not exist. continue; } - dirtyPartitionSet.add(tabletMeta.getPartitionId()); + dirtyPartitionSet.add(tabletMeta.getPhysicalPartitionId()); // Invalid column set should union invalidDictCacheColumns.addAll(finishedTablets.get(i).getInvalidDictCacheColumns()); diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java index c3d60c510769b..4a232b2e42014 100644 --- a/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java +++ b/fe/fe-core/src/main/java/com/starrocks/transaction/OlapTableTxnStateListener.java @@ -108,7 +108,7 @@ public void preCommit(TransactionState txnState, List tabletCo if (tableId != table.getId()) { continue; } - long partitionId = tabletMeta.getPartitionId(); + long partitionId = tabletMeta.getPhysicalPartitionId(); if (table.getPhysicalPartition(partitionId) == null) { // this can happen when partitionId == -1 (tablet being dropping) // or partition really not exist. diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java index b7ff38ed67cb9..76283f66f7725 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/TempPartitionTest.java @@ -145,7 +145,7 @@ private long getPartitionIdByTabletId(long tabletId) { if (tabletMeta == null) { return -1; } - return tabletMeta.getPartitionId(); + return tabletMeta.getPhysicalPartitionId(); } private void getPartitionNameToTabletIdMap(String tbl, boolean isTemp, Map partNameToTabletId) diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java index 5f9e9aeae5e8f..999713dcfae56 100644 --- a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java @@ -23,7 +23,7 @@ import com.starrocks.catalog.MaterializedIndex; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.OlapTable.OlapTableState; -import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.Replica; import com.starrocks.catalog.TabletInvertedIndex; import com.starrocks.catalog.TabletMeta; @@ -380,7 +380,7 @@ public void testHandleMigration() throws TException { db.readUnlock(); } - Partition partition = table.getPartition(tabletMeta.getPartitionId()); + PhysicalPartition partition = table.getPhysicalPartition(tabletMeta.getPhysicalPartitionId()); MaterializedIndex idx = partition.getIndex(tabletMeta.getIndexId()); LocalTablet tablet = (LocalTablet) idx.getTablet(tabletId); From b76cd8a53e8144217c6d65b6b297b5543cf81840 Mon Sep 17 00:00:00 2001 From: meegoo Date: Tue, 29 Oct 2024 07:23:50 +0800 Subject: [PATCH 2/3] fix --- .../java/com/starrocks/catalog/Partition.java | 30 - .../clone/DiskAndTabletLoadReBalancer.java | 17 - .../com/starrocks/clone/TabletChecker.java | 18 +- .../com/starrocks/clone/TabletSchedCtx.java | 16 +- .../com/starrocks/clone/TabletScheduler.java | 33 +- .../consistency/CheckConsistencyJob.java | 15 - .../consistency/ConsistencyChecker.java | 35 -- .../main/java/com/starrocks/lake/Utils.java | 5 - .../com/starrocks/leader/ReportHandler.java | 17 - .../java/com/starrocks/qe/ShowExecutor.java | 529 +----------------- .../com/starrocks/server/LocalMetastore.java | 7 +- .../alter/LakeTableSchemaChangeJobTest.java | 2 +- 12 files changed, 10 insertions(+), 714 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java index ae5c36df6b589..356d81590d308 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Partition.java @@ -648,34 +648,4 @@ public long getMinRetainVersion() { public void setMinRetainVersion(long minRetainVersion) { this.minRetainVersion = minRetainVersion; } -<<<<<<< HEAD -======= - - public String generatePhysicalPartitionName(long physicalPartitionId) { - return this.name + '_' + physicalPartitionId; - } - - @Override - public void gsonPostProcess() throws IOException { - if (dataVersion == 0) { - dataVersion = visibleVersion; - } - if (nextDataVersion == 0) { - nextDataVersion = nextVersion; - } - if (versionEpoch == 0) { - versionEpoch = nextVersionEpoch(); - } - if (versionTxnType == null) { - versionTxnType = TransactionType.TXN_NORMAL; - } - - for (PhysicalPartitionImpl subPartition : idToSubPartition.values()) { - if (subPartition.getName() == null) { - subPartition.setName(generatePhysicalPartitionName(subPartition.getId())); - } - nameToSubPartition.put(subPartition.getName(), subPartition); - } - } ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java index 580c02b719a32..766461e704939 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java @@ -1540,17 +1540,10 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, } try { -<<<<<<< HEAD db.readLock(); - PhysicalPartition partition = globalStateMgr.getPhysicalPartitionIncludeRecycleBin( - olapTable, tabletMeta.getPhysicalPartitionId()); - if (partition == null) { -======= - locker.lockDatabase(db, LockType.READ); PhysicalPartition physicalPartition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId()); if (physicalPartition == null) { ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) return true; } @@ -1564,13 +1557,8 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an return true; } -<<<<<<< HEAD short replicaNum = globalStateMgr - .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); -======= - short replicaNum = globalStateMgr.getLocalMetastore() .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), physicalPartition.getParentId()); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicaNum == (short) -1) { return true; } @@ -1578,13 +1566,8 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an Pair statusPair = TabletChecker.getTabletHealthStatusWithPriority( tablet, -<<<<<<< HEAD GlobalStateMgr.getCurrentSystemInfo(), - partition.getVisibleVersion(), -======= - globalStateMgr.getNodeMgr().getClusterInfo(), physicalPartition.getVisibleVersion(), ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) replicaNum, aliveBeIds, olapTable.getLocation()); diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java index 3bb74d8629816..aa429ef5f68a4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletChecker.java @@ -299,12 +299,8 @@ private void doCheck(boolean isUrgent) { } OlapTable olapTbl = (OlapTable) table; -<<<<<<< HEAD - for (Partition partition : GlobalStateMgr.getCurrentState().getAllPartitionsIncludeRecycleBin(olapTbl)) { -======= - for (PhysicalPartition physicalPartition : GlobalStateMgr.getCurrentState().getLocalMetastore() - .getAllPhysicalPartitionsIncludeRecycleBin(olapTbl)) { ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) + for (PhysicalPartition physicalPartition : GlobalStateMgr.getCurrentState() + .getLocalMetastore().getAllPhysicalPartitionsIncludeRecycleBin(olapTbl)) { partitionChecked++; boolean isPartitionUrgent = isPartitionUrgent(dbId, table.getId(), physicalPartition.getId()); @@ -327,13 +323,8 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an if (GlobalStateMgr.getCurrentState().getTableIncludeRecycleBin(db, olapTbl.getId()) == null) { continue TABLE; } -<<<<<<< HEAD if (GlobalStateMgr.getCurrentState() - .getPartitionIncludeRecycleBin(olapTbl, partition.getId()) == null) { -======= - if (GlobalStateMgr.getCurrentState().getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTbl, physicalPartition.getId()) == null) { ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) continue; } } @@ -350,13 +341,8 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an } short replicaNum = GlobalStateMgr.getCurrentState() -<<<<<<< HEAD - .getReplicationNumIncludeRecycleBin(olapTbl.getPartitionInfo(), partition.getId()); -======= - .getLocalMetastore() .getReplicationNumIncludeRecycleBin( olapTbl.getPartitionInfo(), physicalPartition.getParentId()); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicaNum == (short) -1) { continue; } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java index 317904ea3eff5..6059226e94ea4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletSchedCtx.java @@ -242,17 +242,7 @@ public TabletSchedCtx(Type type, long dbId, long tblId, long physicalPartitionId } @VisibleForTesting -<<<<<<< HEAD - public TabletSchedCtx(Type type, long dbId, long tblId, long partId, - long idxId, long tabletId, long createTime) { - this(type, dbId, tblId, partId, partId, idxId, tabletId, createTime); - } - - @VisibleForTesting - public TabletSchedCtx(Type type, long dbId, long tblId, long partId, -======= public TabletSchedCtx(Type type, long dbId, long tblId, long physicalPartitionId, ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) long idxId, long tabletId, long createTime, SystemInfoService infoService) { this.type = type; this.dbId = dbId; @@ -1049,12 +1039,8 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) } short replicationNum = -<<<<<<< HEAD - globalStateMgr.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partitionId); -======= - globalStateMgr.getLocalMetastore() + globalStateMgr .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId()); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicationNum == (short) -1) { throw new SchedException(Status.UNRECOVERABLE, "invalid replication number"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java index 3e00589272264..e5ce5b650e48a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/TabletScheduler.java @@ -708,44 +708,28 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) OlapTableState tableState = tbl.getState(); -<<<<<<< HEAD - Partition partition = GlobalStateMgr.getCurrentState() - .getPartitionIncludeRecycleBin(tbl, tabletCtx.getPartitionId()); - if (partition == null) { - throw new SchedException(Status.UNRECOVERABLE, "partition does not exist"); - } - - short replicaNum = GlobalStateMgr.getCurrentState() - .getReplicationNumIncludeRecycleBin(tbl.getPartitionInfo(), partition.getId()); -======= PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState() - .getLocalMetastore().getPhysicalPartitionIncludeRecycleBin(tbl, tabletCtx.getPhysicalPartitionId()); + .getPhysicalPartitionIncludeRecycleBin(tbl, tabletCtx.getPhysicalPartitionId()); if (physicalPartition == null) { throw new SchedException(Status.UNRECOVERABLE, "physical partition " + tabletCtx.getPhysicalPartitionId() + "does not exist"); } - Partition logicalPartition = GlobalStateMgr.getCurrentState().getLocalMetastore() + Partition logicalPartition = GlobalStateMgr.getCurrentState() .getPartitionIncludeRecycleBin(tbl, physicalPartition.getParentId()); if (logicalPartition == null) { throw new SchedException(Status.UNRECOVERABLE, "partition " + physicalPartition.getParentId() + "does not exist"); } - short replicaNum = GlobalStateMgr.getCurrentState().getLocalMetastore() + short replicaNum = GlobalStateMgr.getCurrentState() .getReplicationNumIncludeRecycleBin(tbl.getPartitionInfo(), physicalPartition.getParentId()); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (replicaNum == (short) -1) { throw new SchedException(Status.UNRECOVERABLE, "invalid replication number"); } -<<<<<<< HEAD DataProperty dataProperty = GlobalStateMgr.getCurrentState() - .getDataPropertyIncludeRecycleBin(tbl.getPartitionInfo(), partition.getId()); -======= - DataProperty dataProperty = GlobalStateMgr.getCurrentState().getLocalMetastore() .getDataPropertyIncludeRecycleBin(tbl.getPartitionInfo(), physicalPartition.getParentId()); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (dataProperty == null) { throw new SchedException(Status.UNRECOVERABLE, "partition data property not exist"); } @@ -2093,17 +2077,8 @@ private void checkMetaExist(TabletSchedCtx ctx) throws SchedException { throw new SchedException(Status.UNRECOVERABLE, "table " + ctx.getTblId() + " dose not exist"); } -<<<<<<< HEAD - Partition partition = GlobalStateMgr.getCurrentState().getPartitionIncludeRecycleBin(tbl, ctx.getPartitionId()); - if (partition == null) { - throw new SchedException(Status.UNRECOVERABLE, "partition " + ctx.getPartitionId() + " dose not exist"); - } - - PhysicalPartition physicalPartition = partition.getSubPartition(ctx.getPhysicalPartitionId()); -======= - PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState().getLocalMetastore() + PhysicalPartition physicalPartition = GlobalStateMgr.getCurrentState() .getPhysicalPartitionIncludeRecycleBin(tbl, ctx.getPhysicalPartitionId()); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (physicalPartition == null) { throw new SchedException(Status.UNRECOVERABLE, "partition " + ctx.getPhysicalPartitionId() + " dose not exist"); } diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java index a492638f90cd2..75cf3b726332d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/CheckConsistencyJob.java @@ -185,21 +185,12 @@ public boolean sendTasks() { } CheckConsistencyTask task = new CheckConsistencyTask(null, replica.getBackendId(), -<<<<<<< HEAD - tabletMeta.getDbId(), - tabletMeta.getTableId(), - tabletMeta.getPartitionId(), - tabletMeta.getIndexId(), - tabletId, checkedSchemaHash, - checkedVersion); -======= tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(), tabletId, checkedSchemaHash, checkedVersion); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) // add task to send batchTask.addTask(task); @@ -376,15 +367,9 @@ public synchronized int tryFinishJob() { tablet.setCheckedVersion(checkedVersion); // log -<<<<<<< HEAD - ConsistencyCheckInfo info = new ConsistencyCheckInfo(db.getId(), table.getId(), partition.getId(), - index.getId(), tabletId, lastCheckTime, - checkedVersion, isConsistent); -======= ConsistencyCheckInfo info = new ConsistencyCheckInfo(db.getId(), table.getId(), physicalPartition.getId(), index.getId(), tabletId, lastCheckTime, checkedVersion, isConsistent); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) journalTask = GlobalStateMgr.getCurrentState().getEditLog().logFinishConsistencyCheckNoWait(info); } finally { db.writeUnlock(); diff --git a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java index 1b8a2d39f650e..c2fa05d85bee1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/consistency/ConsistencyChecker.java @@ -315,13 +315,8 @@ protected List chooseTablets() { // sort partitions Queue partitionQueue = -<<<<<<< HEAD - new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); - for (PhysicalPartition partition : table.getPhysicalPartitions()) { -======= new PriorityQueue<>(Math.max(table.getAllPhysicalPartitions().size(), 1), COMPARATOR); for (PhysicalPartition physicalPartition : table.getPhysicalPartitions()) { ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) // check partition's replication num. if 1 replication. skip if (table.getPartitionInfo().getReplicationNum(physicalPartition.getParentId()) == (short) 1) { LOG.debug("partition[{}]'s replication num is 1. ignore", physicalPartition.getParentId()); @@ -329,15 +324,9 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an } // check if this partition has no data -<<<<<<< HEAD - if (partition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { - LOG.debug("partition[{}]'s version is {}. ignore", partition.getId(), - Partition.PARTITION_INIT_VERSION); -======= if (physicalPartition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION) { LOG.debug("partition[{}]'s version is {}. ignore", physicalPartition.getId(), Partition.PARTITION_INIT_VERSION); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) continue; } if (physicalPartition instanceof Partition) { @@ -352,11 +341,7 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an // sort materializedIndices List visibleIndexes = -<<<<<<< HEAD - partition.getMaterializedIndices(IndexExtState.VISIBLE); -======= physicalPartition.getMaterializedIndices(IndexExtState.VISIBLE); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) Queue indexQueue = new PriorityQueue<>(Math.max(visibleIndexes.size(), 1), COMPARATOR); indexQueue.addAll(visibleIndexes); @@ -381,20 +366,12 @@ >>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id an if (physicalPartition.getVisibleVersion() == tablet.getCheckedVersion()) { if (tablet.isConsistent()) { LOG.debug("tablet[{}]'s version[{}-{}] has been checked. ignore", -<<<<<<< HEAD - chosenTabletId, tablet.getCheckedVersion(), partition.getVisibleVersion()); - } - } else { - LOG.info("chose tablet[{}-{}-{}-{}-{}] to check consistency", db.getId(), - table.getId(), partition.getId(), index.getId(), chosenTabletId); -======= chosenTabletId, tablet.getCheckedVersion(), physicalPartition.getVisibleVersion()); } } else { LOG.info("chose tablet[{}-{}-{}-{}-{}] to check consistency", db.getId(), table.getId(), physicalPartition.getId(), index.getId(), chosenTabletId); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) chosenTablets.add(chosenTabletId); } @@ -440,7 +417,6 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM LOG.warn("replay finish consistency check failed, db is null, info: {}", info); return; } -<<<<<<< HEAD db.writeLock(); try { OlapTable table = (OlapTable) db.getTable(info.getTableId()); @@ -448,20 +424,9 @@ public void replayFinishConsistencyCheck(ConsistencyCheckInfo info, GlobalStateM LOG.warn("replay finish consistency check failed, table is null, info: {}", info); return; } - Partition partition = table.getPartition(info.getPartitionId()); - if (partition == null) { -======= - OlapTable table = (OlapTable) db.getTable(info.getTableId()); - if (table == null) { - LOG.warn("replay finish consistency check failed, table is null, info: {}", info); - return; - } - try (AutoCloseableLock ignore - = new AutoCloseableLock(new Locker(), db, Lists.newArrayList(table.getId()), LockType.WRITE)) { PhysicalPartition physicalPartition = table.getPhysicalPartition(info.getPhysicalPartitionId()); if (physicalPartition == null) { ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) LOG.warn("replay finish consistency check failed, partition is null, info: {}", info); return; } diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java b/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java index 0b448bf10c8f0..6350f726fb2d8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/Utils.java @@ -86,13 +86,8 @@ public static Map> groupTabletID(Collection partitio throws NoAliveBackendException { Map> groupMap = new HashMap<>(); for (Partition partition : partitions) { -<<<<<<< HEAD - for (PhysicalPartition physicalParition : partition.getSubPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices(indexState)) { -======= for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { for (MaterializedIndex index : physicalPartition.getMaterializedIndices(indexState)) { ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) for (Tablet tablet : index.getTablets()) { Long beId = chooseBackend((LakeTablet) tablet); if (beId == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java index 80dae224f1624..bc20a5bc4d995 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java @@ -1366,15 +1366,11 @@ private static void handleSetTabletInMemory(long backendId, Map b } db.readLock(); try { -<<<<<<< HEAD OlapTable olapTable = (OlapTable) db.getTable(tableId); if (olapTable == null) { continue; } - Partition partition = olapTable.getPartition(partitionId); -======= PhysicalPartition partition = olapTable.getPhysicalPartition(partitionId); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) if (partition == null) { continue; } @@ -1736,24 +1732,11 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon } db.writeLock(); try { -<<<<<<< HEAD OlapTable olapTable = (OlapTable) globalStateMgr.getTableIncludeRecycleBin(db, tableId); if (olapTable == null) { throw new MetaNotFoundException("table[" + tableId + "] does not exist"); } - if (globalStateMgr.getLocalMetastore().getPartitionIncludeRecycleBin(olapTable, partitionId) == null) { - throw new MetaNotFoundException("partition[" + partitionId + "] does not exist"); - } - short replicationNum = - globalStateMgr.getLocalMetastore() - .getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partitionId); - if (replicationNum == (short) -1) { - throw new MetaNotFoundException("invalid replication number of partition [" + partitionId + "]"); - } - -======= ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) PhysicalPartition partition = globalStateMgr.getLocalMetastore() .getPhysicalPartitionIncludeRecycleBin(olapTable, physicalPartitionId); if (partition == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index 5d1bcbad81278..70c266c216591 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -504,533 +504,6 @@ private void handleShowMaterializedView() throws AnalysisException { Authorizer.checkAnyActionOnMaterializedView(connectContext.getCurrentUserIdentity(), connectContext.getCurrentRoleIds(), new TableName(db.getFullName(), mvTable.getName())); } catch (AccessDeniedException e) { -<<<<<<< HEAD -======= - AccessDeniedException.reportAccessDenied(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, - context.getCurrentUserIdentity(), - context.getCurrentRoleIds(), - PrivilegeType.ANY.name(), ObjectType.TABLE.name(), tableName); - } - - Table table = MetaUtils.getSessionAwareTable(context, db, new TableName(dbName, tableName)); - if (table == null) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); - } - - if (!table.isNativeTableOrMaterializedView()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE, tableName); - } - - OlapTable olapTable = (OlapTable) table; - int i = 0; - long totalSize = 0; - long totalReplicaCount = 0; - - // sort by index name - Map indexNames = olapTable.getIndexNameToId(); - Map sortedIndexNames = new TreeMap<>(indexNames); - - for (Long indexId : sortedIndexNames.values()) { - long indexSize = 0; - long indexReplicaCount = 0; - long indexRowCount = 0; - for (PhysicalPartition partition : olapTable.getAllPhysicalPartitions()) { - MaterializedIndex mIndex = partition.getIndex(indexId); - indexSize += mIndex.getDataSize(); - indexReplicaCount += mIndex.getReplicaCount(); - indexRowCount += mIndex.getRowCount(); - } - - Pair indexSizePair = DebugUtil.getByteUint(indexSize); - String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(indexSizePair.first) + " " - + indexSizePair.second; - - List row = null; - if (i == 0) { - row = Arrays.asList(tableName, - olapTable.getIndexNameById(indexId), - readableSize, String.valueOf(indexReplicaCount), - String.valueOf(indexRowCount)); - } else { - row = Arrays.asList("", - olapTable.getIndexNameById(indexId), - readableSize, String.valueOf(indexReplicaCount), - String.valueOf(indexRowCount)); - } - - totalSize += indexSize; - totalReplicaCount += indexReplicaCount; - totalRows.add(row); - - i++; - } // end for indices - - Pair totalSizePair = DebugUtil.getByteUint(totalSize); - String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalSizePair.first) + " " - + totalSizePair.second; - List row = Arrays.asList("", "Total", readableSize, String.valueOf(totalReplicaCount), ""); - totalRows.add(row); - } - } catch (AnalysisException e) { - throw new SemanticException(e.getMessage()); - } finally { - locker.unLockDatabase(db, LockType.READ); - } - return new ShowResultSet(statement.getMetaData(), statement.getResultRows()); - } - - @Override - public ShowResultSet visitShowCollationStatement(ShowCollationStmt statement, ConnectContext context) { - List> rows = Lists.newArrayList(); - List row = Lists.newArrayList(); - // | utf8_general_ci | utf8 | 33 | Yes | Yes | 1 | - row.add("utf8_general_ci"); - row.add("utf8"); - row.add("33"); - row.add("Yes"); - row.add("Yes"); - row.add("1"); - rows.add(row); - // | binary | binary | 63 | Yes | Yes | 1 | - row = Lists.newArrayList(); - row.add("binary"); - row.add("binary"); - row.add("63"); - row.add("Yes"); - row.add("Yes"); - row.add("1"); - rows.add(row); - // | gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | - row = Lists.newArrayList(); - row.add("gbk_chinese_ci"); - row.add("gbk"); - row.add("28"); - row.add("Yes"); - row.add("Yes"); - row.add("1"); - rows.add(row); - return new ShowResultSet(statement.getMetaData(), rows); - } - - @Override - public ShowResultSet visitShowPartitionsStatement(ShowPartitionsStmt statement, ConnectContext context) { - ProcNodeInterface procNodeI = statement.getNode(); - Preconditions.checkNotNull(procNodeI); - try { - List> rows = ((PartitionsProcDir) procNodeI).fetchResultByFilter(statement.getFilterMap(), - statement.getOrderByPairs(), statement.getLimitElement()).getRows(); - return new ShowResultSet(statement.getMetaData(), rows); - } catch (AnalysisException e) { - throw new SemanticException(e.getMessage()); - } - } - - @Override - public ShowResultSet visitShowTabletStatement(ShowTabletStmt statement, ConnectContext context) { - List> rows = Lists.newArrayList(); - - GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - if (statement.isShowSingleTablet()) { - long tabletId = statement.getTabletId(); - TabletInvertedIndex invertedIndex = GlobalStateMgr.getCurrentState().getTabletInvertedIndex(); - TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId); - Long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; - String dbName = null; - Long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; - String tableName = null; - Long partitionId = tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; - String partitionName = null; - Long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; - String indexName = null; - Boolean isSync = true; - - // check real meta - do { - Database db = globalStateMgr.getDb(dbId); - if (db == null) { - isSync = false; - break; - } - dbName = db.getFullName(); - - Locker locker = new Locker(); - locker.lockDatabase(db, LockType.READ); - try { - Table table = db.getTable(tableId); - if (!(table instanceof OlapTable)) { - isSync = false; - break; - } - tableName = table.getName(); - Pair privResult = Authorizer.checkPrivForShowTablet(context, dbName, table); - if (!privResult.first) { - AccessDeniedException.reportAccessDenied( - InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, - context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - PrivilegeType.ANY.name(), ObjectType.TABLE.name(), null); - } - - OlapTable olapTable = (OlapTable) table; - PhysicalPartition physicalPartition = olapTable.getPhysicalPartition(partitionId); - if (physicalPartition == null) { - isSync = false; - break; - } - Partition partition = olapTable.getPartition(physicalPartition.getParentId()); - if (partition == null) { - isSync = false; - break; - } - partitionName = partition.getName(); - - MaterializedIndex index = physicalPartition.getIndex(indexId); - if (index == null) { - isSync = false; - break; - } - indexName = olapTable.getIndexNameById(indexId); - - if (table.isCloudNativeTableOrMaterializedView()) { - break; - } - - LocalTablet tablet = (LocalTablet) index.getTablet(tabletId); - if (tablet == null) { - isSync = false; - break; - } - - List replicas = tablet.getImmutableReplicas(); - for (Replica replica : replicas) { - Replica tmp = invertedIndex.getReplica(tabletId, replica.getBackendId()); - if (tmp == null) { - isSync = false; - break; - } - // use !=, not equals(), because this should be the same object. - if (tmp != replica) { - isSync = false; - break; - } - } - - } finally { - locker.unLockDatabase(db, LockType.READ); - } - } while (false); - - String detailCmd = String.format("SHOW PROC '/dbs/%d/%d/partitions/%d/%d/%d';", - dbId, tableId, partitionId, indexId, tabletId); - rows.add(Lists.newArrayList(dbName, tableName, partitionName, indexName, - dbId.toString(), tableId.toString(), - partitionId.toString(), indexId.toString(), - isSync.toString(), detailCmd)); - } else { - Database db = globalStateMgr.getDb(statement.getDbName()); - MetaUtils.checkDbNullAndReport(db, statement.getDbName()); - - Locker locker = new Locker(); - locker.lockDatabase(db, LockType.READ); - try { - Table table = MetaUtils.getSessionAwareTable( - context, db, new TableName(statement.getDbName(), statement.getTableName())); - if (table == null) { - ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, statement.getTableName()); - } - if (!table.isNativeTableOrMaterializedView()) { - ErrorReport.reportSemanticException(ErrorCode.ERR_NOT_OLAP_TABLE, statement.getTableName()); - } - - Pair privResult = Authorizer.checkPrivForShowTablet( - context, db.getFullName(), table); - if (!privResult.first) { - AccessDeniedException.reportAccessDenied( - InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, - context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - PrivilegeType.ANY.name(), ObjectType.TABLE.name(), null); - } - Boolean hideIpPort = privResult.second; - statement.setTable(table); - - OlapTable olapTable = (OlapTable) table; - long sizeLimit = -1; - if (statement.hasOffset() && statement.hasLimit()) { - sizeLimit = statement.getOffset() + statement.getLimit(); - } else if (statement.hasLimit()) { - sizeLimit = statement.getLimit(); - } - boolean stop = false; - Collection partitions = new ArrayList<>(); - if (statement.hasPartition()) { - PartitionNames partitionNames = statement.getPartitionNames(); - for (String partName : partitionNames.getPartitionNames()) { - Partition partition = olapTable.getPartition(partName, partitionNames.isTemp()); - if (partition == null) { - throw new SemanticException("Unknown partition: " + partName); - } - partitions.add(partition); - } - } else { - partitions = olapTable.getPartitions(); - } - List> tabletInfos = new ArrayList<>(); - String indexName = statement.getIndexName(); - long indexId = -1; - if (indexName != null) { - Long id = olapTable.getIndexIdByName(indexName); - if (id == null) { - // invalid indexName - ErrorReport.reportSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, statement.getIndexName()); - } - indexId = id; - } - for (Partition partition : partitions) { - if (stop) { - break; - } - for (PhysicalPartition physicalPartition : partition.getSubPartitions()) { - for (MaterializedIndex index : physicalPartition.getMaterializedIndices(IndexExtState.ALL)) { - if (indexId > -1 && index.getId() != indexId) { - continue; - } - if (olapTable.isCloudNativeTableOrMaterializedView()) { - LakeTabletsProcDir procNode = new LakeTabletsProcDir(db, olapTable, index); - tabletInfos.addAll(procNode.fetchComparableResult()); - } else { - LocalTabletsProcDir procDir = new LocalTabletsProcDir(db, olapTable, index); - tabletInfos.addAll(procDir.fetchComparableResult( - statement.getVersion(), statement.getBackendId(), statement.getReplicaState(), - hideIpPort)); - } - if (sizeLimit > -1 && CollectionUtils.isEmpty(statement.getOrderByPairs()) - && tabletInfos.size() >= sizeLimit) { - stop = true; - break; - } - } - } - } - - // order by - List orderByPairs = statement.getOrderByPairs(); - ListComparator> comparator; - if (orderByPairs != null) { - OrderByPair[] orderByPairArr = new OrderByPair[orderByPairs.size()]; - comparator = new ListComparator<>(orderByPairs.toArray(orderByPairArr)); - } else { - // order by tabletId, replicaId - comparator = new ListComparator<>(0, 1); - } - tabletInfos.sort(comparator); - - if (sizeLimit > -1 && tabletInfos.size() >= sizeLimit) { - tabletInfos = tabletInfos.subList((int) statement.getOffset(), (int) sizeLimit); - } - - for (List tabletInfo : tabletInfos) { - List oneTablet = new ArrayList<>(tabletInfo.size()); - for (Comparable column : tabletInfo) { - oneTablet.add(column.toString()); - } - rows.add(oneTablet); - } - } finally { - locker.unLockDatabase(db, LockType.READ); - } - } - - return new ShowResultSet(statement.getMetaData(), rows); - } - - @Override - public ShowResultSet visitShowBackupStatement(ShowBackupStmt statement, ConnectContext context) { - Database filterDb = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); - List> infos = Lists.newArrayList(); - List dbs = Lists.newArrayList(); - - if (filterDb == null) { - for (Map.Entry entry : GlobalStateMgr.getCurrentState() - .getLocalMetastore().getIdToDb().entrySet()) { - dbs.add(entry.getValue()); - } - } else { - dbs.add(filterDb); - } - - for (Database db : dbs) { - AbstractJob jobI = GlobalStateMgr.getCurrentState().getBackupHandler().getJob(db.getId()); - if (jobI == null || !(jobI instanceof BackupJob)) { - // show next db - continue; - } - - BackupJob backupJob = (BackupJob) jobI; - - // check privilege - List tableRefs = backupJob.getTableRef(); - AtomicBoolean privilegeDeny = new AtomicBoolean(false); - tableRefs.forEach(tableRef -> { - TableName tableName = tableRef.getName(); - try { - Authorizer.checkTableAction(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), - tableName.getDb(), tableName.getTbl(), PrivilegeType.EXPORT); - } catch (AccessDeniedException e) { - privilegeDeny.set(true); - } - }); - if (privilegeDeny.get()) { - return new ShowResultSet(statement.getMetaData(), EMPTY_SET); - } - - List info = backupJob.getInfo(); - infos.add(info); - } - return new ShowResultSet(statement.getMetaData(), infos); - } - - @Override - public ShowResultSet visitShowRestoreStatement(ShowRestoreStmt statement, ConnectContext context) { - Database filterDb = GlobalStateMgr.getCurrentState().getDb(statement.getDbName()); - List> infos = Lists.newArrayList(); - List dbs = Lists.newArrayList(); - - if (filterDb == null) { - for (Map.Entry entry : GlobalStateMgr.getCurrentState() - .getLocalMetastore().getIdToDb().entrySet()) { - dbs.add(entry.getValue()); - } - } else { - dbs.add(filterDb); - } - - for (Database db : dbs) { - AbstractJob jobI = GlobalStateMgr.getCurrentState().getBackupHandler().getJob(db.getId()); - if (jobI == null || !(jobI instanceof RestoreJob)) { - // show next db - continue; - } - - RestoreJob restoreJob = (RestoreJob) jobI; - List info = restoreJob.getInfo(); - infos.add(info); - } - return new ShowResultSet(statement.getMetaData(), infos); - } - - @Override - public ShowResultSet visitShowBrokerStatement(ShowBrokerStmt statement, ConnectContext context) { - List> rowSet = GlobalStateMgr.getCurrentState().getBrokerMgr().getBrokersInfo(); - - // Only success - return new ShowResultSet(statement.getMetaData(), rowSet); - } - - @Override - public ShowResultSet visitShowResourceStatement(ShowResourcesStmt statement, ConnectContext context) { - List> rowSet = GlobalStateMgr.getCurrentState().getResourceMgr().getResourcesInfo(); - - // Only success - return new ShowResultSet(statement.getMetaData(), rowSet); - } - - @Override - public ShowResultSet visitShowExportStatement(ShowExportStmt statement, ConnectContext context) { - GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState(); - Database db = globalStateMgr.getDb(statement.getDbName()); - MetaUtils.checkDbNullAndReport(db, statement.getDbName()); - long dbId = db.getId(); - - ExportMgr exportMgr = globalStateMgr.getExportMgr(); - - Set states = null; - ExportJob.JobState state = statement.getJobState(); - if (state != null) { - states = Sets.newHashSet(state); - } - List> infos = exportMgr.getExportJobInfosByIdOrState( - dbId, statement.getJobId(), states, statement.getQueryId(), - statement.getOrderByPairs(), statement.getLimit()); - - return new ShowResultSet(statement.getMetaData(), infos); - } - - @Override - public ShowResultSet visitShowBackendsStatement(ShowBackendsStmt statement, ConnectContext context) { - List> backendInfos = BackendsProcDir.getClusterBackendInfos(); - return new ShowResultSet(statement.getMetaData(), backendInfos); - } - - @Override - public ShowResultSet visitShowFrontendsStatement(ShowFrontendsStmt statement, ConnectContext context) { - List> infos = Lists.newArrayList(); - FrontendsProcNode.getFrontendsInfo(GlobalStateMgr.getCurrentState(), infos); - return new ShowResultSet(statement.getMetaData(), infos); - } - - @Override - public ShowResultSet visitShowRepositoriesStatement(ShowRepositoriesStmt statement, ConnectContext context) { - List> repoInfos = GlobalStateMgr.getCurrentState().getBackupHandler().getRepoMgr().getReposInfo(); - return new ShowResultSet(statement.getMetaData(), repoInfos); - } - - @Override - public ShowResultSet visitShowSnapshotStatement(ShowSnapshotStmt statement, ConnectContext context) { - Repository repo = GlobalStateMgr.getCurrentState().getBackupHandler().getRepoMgr().getRepo(statement.getRepoName()); - if (repo == null) { - throw new SemanticException("Repository " + statement.getRepoName() + " does not exist"); - } - - List> snapshotInfos = repo.getSnapshotInfos(statement.getSnapshotName(), statement.getTimestamp(), - statement.getSnapshotNames()); - return new ShowResultSet(statement.getMetaData(), snapshotInfos); - } - - @Override - public ShowResultSet visitShowGrantsStatement(ShowGrantsStmt statement, ConnectContext context) { - AuthorizationMgr authorizationManager = GlobalStateMgr.getCurrentState().getAuthorizationMgr(); - try { - List> infos = new ArrayList<>(); - if (statement.getRole() != null) { - List granteeRole = authorizationManager.getGranteeRoleDetailsForRole(statement.getRole()); - if (granteeRole != null) { - infos.add(granteeRole); - } - - Map> typeToPrivilegeEntryList = - authorizationManager.getTypeToPrivilegeEntryListByRole(statement.getRole()); - infos.addAll(privilegeToRowString(authorizationManager, - new GrantRevokeClause(null, statement.getRole()), typeToPrivilegeEntryList)); - } else { - List granteeRole = authorizationManager.getGranteeRoleDetailsForUser(statement.getUserIdent()); - if (granteeRole != null) { - infos.add(granteeRole); - } - - Map> typeToPrivilegeEntryList = - authorizationManager.getTypeToPrivilegeEntryListByUser(statement.getUserIdent()); - infos.addAll(privilegeToRowString(authorizationManager, - new GrantRevokeClause(statement.getUserIdent(), null), typeToPrivilegeEntryList)); - } - return new ShowResultSet(statement.getMetaData(), infos); - } catch (PrivilegeException e) { - throw new SemanticException(e.getMessage()); - } - } - - private List> privilegeToRowString(AuthorizationMgr authorizationManager, GrantRevokeClause userOrRoleName, - Map> typeToPrivilegeEntryList) - throws PrivilegeException { - List> infos = new ArrayList<>(); - for (Map.Entry> typeToPrivilegeEntry - : typeToPrivilegeEntryList.entrySet()) { - for (PrivilegeEntry privilegeEntry : typeToPrivilegeEntry.getValue()) { - ObjectType objectType = typeToPrivilegeEntry.getKey(); - String catalogName; - try { - catalogName = getCatalogNameFromPEntry(objectType, privilegeEntry); - } catch (MetaNotFoundException e) { - // ignore this entry ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) continue; } @@ -2445,7 +1918,7 @@ private void handleShowTablet() throws AnalysisException { String dbName = null; Long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; String tableName = null; - Long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; + Long partitionId = tabletMeta != null ? tabletMeta.getPhysicalPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; String partitionName = null; Long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; String indexName = null; diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index adba96fca9079..b607a65fae0b1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -1695,14 +1695,9 @@ private PhysicalPartition createPhysicalPartition(Database db, OlapTable olapTab shardGroupId = GlobalStateMgr.getCurrentStarOSAgent(). createShardGroup(db.getId(), olapTable.getId(), id); } -<<<<<<< HEAD - PhysicalPartitionImpl physicalParition = new PhysicalPartitionImpl( - id, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); -======= PhysicalPartitionImpl physicalPartition = new PhysicalPartitionImpl( - id, name, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); ->>>>>>> bf04f84df6 ([BugFix] Fix tablet meta use tabletMeta uses partition_id and physica… (#52373)) + id, partition.getId(), shardGroupId, indexMap.get(olapTable.getBaseIndexId())); PartitionInfo partitionInfo = olapTable.getPartitionInfo(); short replicationNum = partitionInfo.getReplicationNum(partitionId); diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java index 3dd96cb35d075..0500f053c194a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/LakeTableSchemaChangeJobTest.java @@ -914,7 +914,7 @@ public void dropTabletAndDeleteShard(List shardIds, StarOSAgent starOSAgen Assert.assertTrue(tabletMeta.isLakeTablet()); Assert.assertEquals(db.getId(), tabletMeta.getDbId()); Assert.assertEquals(table.getId(), tabletMeta.getTableId()); - Assert.assertEquals(partition.getId(), tabletMeta.getPartitionId()); + Assert.assertEquals(partition.getId(), tabletMeta.getPhysicalPartitionId()); Assert.assertEquals(normalIndex.getId(), tabletMeta.getIndexId()); } From ac7e73690fde875c8e7ff660e9d58558120f58f2 Mon Sep 17 00:00:00 2001 From: meegoo Date: Wed, 30 Oct 2024 04:35:43 +0800 Subject: [PATCH 3/3] fix --- .../java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java index 766461e704939..19cf1a82f44d7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java @@ -1541,7 +1541,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId, try { db.readLock(); - PhysicalPartition physicalPartition = globalStateMgr.getLocalMetastore() + PhysicalPartition physicalPartition = globalStateMgr .getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId()); if (physicalPartition == null) { return true;