Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion(backport #52258) (backport #52373) #52375

Merged
merged 3 commits into from
Oct 29, 2024
Merged

[BugFix] Fix tablet meta use tabletMeta uses partition_id and physical_partition_id at the same time to cause confusion(backport #52258) (backport #52373) #52375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9ed6e14
[BugFix] Fix tablet meta use tabletMeta uses partition_id and physica…
meegoo Oct 28, 2024
b76cd8a
fix
meegoo Oct 28, 2024
ac7e736
fix
meegoo Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -825,7 +825,7 @@ private void onFinished(OlapTable tbl) {
}
// replace the origin index with shadow index, set index state as NORMAL
for (Partition partition : tbl.getPartitions()) {
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getParentId()).getStorageMedium();
TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partition.getId()).getStorageMedium();
// drop the origin index from partitions
for (Map.Entry<Long, Long> entry : indexIdMap.entrySet()) {
long shadowIdxId = entry.getKey();
Expand Down
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,14 @@ public synchronized List<Partition> getPartitions(long tableId) {
.collect(Collectors.toList());
}

public synchronized List<PhysicalPartition> getPhysicalPartitions(long tableId) {
return idToPartition.values().stream()
.filter(v -> (v.getTableId() == tableId))
.map(RecyclePartitionInfo::getPartition)
.flatMap(p -> p.getSubPartitions().stream())
.collect(Collectors.toList());
}

/**
* if we can erase this instance, we should check if anyone enable erase later.
* Only used by main loop.
Expand Down
18 changes: 9 additions & 9 deletions fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
replica.setLastReportVersion(backendTabletInfo.getVersion());

// check if tablet needs migration
long partitionId = tabletMeta.getPartitionId();
long partitionId = tabletMeta.getPhysicalPartitionId();
TStorageMedium storageMedium = storageMediumMap.get(partitionId);
if (storageMedium != null && backendTabletInfo.isSetStorage_medium()) {
if (storageMedium != backendTabletInfo.getStorage_medium()) {
Expand Down Expand Up @@ -239,7 +239,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId);
if (transactionState == null ||
transactionState.getTransactionStatus() == TransactionStatus.ABORTED) {
transactionsToClear.put(transactionId, tabletMeta.getPartitionId());
transactionsToClear.put(transactionId, partitionId);
LOG.debug("transaction id [{}] is not valid any more, "
+ "clear it from backend [{}]", transactionId, backendId);
} else if (transactionState.getTransactionStatus() ==
Expand All @@ -266,7 +266,7 @@ public void tabletReport(long backendId, Map<Long, TTablet> backendTablets,
transactionState.getTransactionId());
} else {
TPartitionVersionInfo versionInfo =
new TPartitionVersionInfo(tabletMeta.getPartitionId(),
new TPartitionVersionInfo(partitionId,
partitionCommitInfo.getVersion(), 0);
Map<Long, Map<Long, TPartitionVersionInfo>> txnMap =
transactionsToPublish.computeIfAbsent(
Expand Down Expand Up @@ -415,11 +415,11 @@ public void checkTabletMetaConsistency(Map<Long, Integer> creatingTableIds) {
}

// validate partition
long partitionId = tabletMeta.getPartitionId();
PhysicalPartition partition = table.getPhysicalPartition(partitionId);
if (partition == null) {
partition = recycleBin.getPhysicalPartition(partitionId);
if (partition != null) {
long partitionId = tabletMeta.getPhysicalPartitionId();
PhysicalPartition physicalPartition = table.getPhysicalPartition(partitionId);
if (physicalPartition == null) {
physicalPartition = recycleBin.getPhysicalPartition(partitionId);
if (physicalPartition != null) {
isInRecycleBin = true;
} else {
deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId,
Expand All @@ -431,7 +431,7 @@ public void checkTabletMetaConsistency(Map<Long, Integer> creatingTableIds) {

// validate index
long indexId = tabletMeta.getIndexId();
MaterializedIndex index = partition.getIndex(indexId);
MaterializedIndex index = physicalPartition.getIndex(indexId);
if (index == null) {
deleteTabletByConsistencyChecker(tabletMeta, tabletId, backendId,
"materialized index " + dbId + "." + tableId + "." +
Expand Down
19 changes: 3 additions & 16 deletions fe/fe-core/src/main/java/com/starrocks/catalog/TabletMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
public class TabletMeta {
private final long dbId;
private final long tableId;
private final long partitionId;
private final long physicalPartitionId;
private final long indexId;

Expand All @@ -59,11 +58,10 @@ public class TabletMeta {

private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();

public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartitionId, long indexId, int schemaHash,
public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash,
TStorageMedium storageMedium, boolean isLakeTablet) {
this.dbId = dbId;
this.tableId = tableId;
this.partitionId = partitionId;
this.physicalPartitionId = physicalPartitionId;
this.indexId = indexId;

Expand All @@ -75,15 +73,9 @@ public TabletMeta(long dbId, long tableId, long partitionId, long physicalPartit
this.isLakeTablet = isLakeTablet;
}

// for single physical partition, the physicalPartitionId is same as partitionId
public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash,
TStorageMedium storageMedium, boolean isLakeTablet) {
this(dbId, tableId, partitionId, partitionId, indexId, schemaHash, storageMedium, isLakeTablet);
}

public TabletMeta(long dbId, long tableId, long partitionId, long indexId, int schemaHash,
public TabletMeta(long dbId, long tableId, long physicalPartitionId, long indexId, int schemaHash,
TStorageMedium storageMedium) {
this(dbId, tableId, partitionId, indexId, schemaHash, storageMedium, false);
this(dbId, tableId, physicalPartitionId, indexId, schemaHash, storageMedium, false);
}

public long getDbId() {
Expand All @@ -94,10 +86,6 @@ public long getTableId() {
return tableId;
}

public long getPartitionId() {
return partitionId;
}

public long getPhysicalPartitionId() {
return physicalPartitionId;
}
Expand Down Expand Up @@ -164,7 +152,6 @@ public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("dbId=").append(dbId);
sb.append(" tableId=").append(tableId);
sb.append(" partitionId=").append(partitionId);
sb.append(" physicalPartitionId=").append(physicalPartitionId);
sb.append(" indexId=").append(indexId);
sb.append(" oldSchemaHash=").append(oldSchemaHash);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/com/starrocks/clone/ColocateTableBalancer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -820,7 +820,7 @@ private ColocateMatchResult doMatchOneGroup(GroupId groupId,
TabletSchedCtx.Type.REPAIR,
// physical partition id is same as partition id
// since colocate table should have only one physical partition
db.getId(), tableId, partition.getId(), partition.getId(),
db.getId(), tableId, partition.getId(),
index.getId(), tablet.getId(),
System.currentTimeMillis());
// the tablet status will be checked and set again when being scheduled
Expand Down
26 changes: 15 additions & 11 deletions fe/fe-core/src/main/java/com/starrocks/clone/DiskAndTabletLoadReBalancer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,9 +544,13 @@ private List<TabletSchedCtx> balanceClusterDisk(ClusterLoadStatistic clusterStat
if (olapTable == null) {
continue;
}
PhysicalPartition physicalPartition = olapTable.getPartition(tabletMeta.getPhysicalPartitionId());
if (physicalPartition == null) {
continue;
}

if (isDestBackendLocationMismatch(olapTable, hBackend.getId(), lBackend.getId(),
tabletMeta.getPartitionId(), tabletId)) {
physicalPartition.getParentId(), tabletId)) {
continue;
}

Expand Down Expand Up @@ -584,7 +588,7 @@ private List<TabletSchedCtx> balanceClusterDisk(ClusterLoadStatistic clusterStat
hState.minusUsedCapacity(replica.getPathHash(), replica.getDataSize());

TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE,
tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getDbId(), tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId(),
tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -800,7 +804,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent,
// NOTICE: state has been changed, the tablet must be selected
destPathUsedCap += replica.getDataSize();
srcPathUsedCap -= replica.getDataSize();
Pair<Long, Long> p = Pair.create(tabletMeta.getPartitionId(), tabletMeta.getIndexId());
Pair<Long, Long> p = Pair.create(tabletMeta.getPhysicalPartitionId(), tabletMeta.getIndexId());
// p: partition <physicalPartitionId, indexId>
// k: partition same to p
srcPathPartitionTablets.compute(p, (k, pTablets) -> {
Expand All @@ -819,7 +823,7 @@ private void balanceBackendDisk(TStorageMedium medium, double avgUsedPercent,

TabletSchedCtx schedCtx =
new TabletSchedCtx(TabletSchedCtx.Type.BALANCE, tabletMeta.getDbId(),
tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(),
tabletMeta.getIndexId(), tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -1414,7 +1418,7 @@ private TabletSchedCtx tryToBalanceTablet(Pair<Long, Set<Long>> srcTablets,
}

TabletSchedCtx schedCtx = new TabletSchedCtx(TabletSchedCtx.Type.BALANCE,
tabletMeta.getDbId(), tabletMeta.getTableId(), tabletMeta.getPartitionId(),
tabletMeta.getDbId(), tabletMeta.getTableId(),
tabletMeta.getPhysicalPartitionId(),
tabletMeta.getIndexId(), tabletId, System.currentTimeMillis());
schedCtx.setOrigPriority(TabletSchedCtx.Priority.LOW);
Expand Down Expand Up @@ -1537,13 +1541,13 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,

try {
db.readLock();
PhysicalPartition partition = globalStateMgr.getPhysicalPartitionIncludeRecycleBin(
olapTable, tabletMeta.getPhysicalPartitionId());
if (partition == null) {
PhysicalPartition physicalPartition = globalStateMgr
.getPhysicalPartitionIncludeRecycleBin(olapTable, tabletMeta.getPhysicalPartitionId());
if (physicalPartition == null) {
return true;
}

MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId());
MaterializedIndex index = physicalPartition.getIndex(tabletMeta.getIndexId());
if (index == null) {
return true;
}
Expand All @@ -1554,7 +1558,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
}

short replicaNum = globalStateMgr
.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), partition.getParentId());
.getReplicationNumIncludeRecycleBin(olapTable.getPartitionInfo(), physicalPartition.getParentId());
if (replicaNum == (short) -1) {
return true;
}
Expand All @@ -1563,7 +1567,7 @@ private boolean isTabletUnhealthy(long dbId, OlapTable olapTable, Long tabletId,
TabletChecker.getTabletHealthStatusWithPriority(
tablet,
GlobalStateMgr.getCurrentSystemInfo(),
partition.getVisibleVersion(),
physicalPartition.getVisibleVersion(),
replicaNum,
aliveBeIds,
olapTable.getLocation());
Expand Down
Loading
Loading