Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BugFix] Fix cherry pick codes conflicts (backport #54688) #54689

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,15 @@ private MVCompensation getMVCompensationForExternal(Table refBaseTable,
try {
ScanOperatorPredicates scanOperatorPredicates = refScanOperator.getScanOperatorPredicates();
Collection<Long> selectPartitionIds = scanOperatorPredicates.getSelectedPartitionIds();
List<PartitionKey> selectPartitionKeys = scanOperatorPredicates.getSelectedPartitionKeys();
// NOTE: ref base table's partition keys may contain multi columns, but mv may only contain one column.
List<Integer> colIndexes = PartitionUtil.getRefBaseTablePartitionColumIndexes(mv, refBaseTable);
if (colIndexes == null) {
return MVCompensation.createUnkownState(sessionVariable);
}
List<PartitionKey> selectPartitionKeys = scanOperatorPredicates.getSelectedPartitionKeys()
.stream()
.map(partitionKey -> PartitionUtil.getSelectedPartitionKey(partitionKey, colIndexes))
.collect(Collectors.toList());
// For scan operator which support prune partitions with OptExternalPartitionPruner,
// we could only compensate partitions which selected partitions need to refresh.
if (SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES.contains(refScanOperator.getOpType())) {
Expand All @@ -344,16 +352,7 @@ private MVCompensation getMVCompensationForExternal(Table refBaseTable,
return MVCompensation.createUnkownState(sessionVariable);
}
}

// NOTE: ref base table's partition keys may contain multi columns, but mv may only contain one column.
List<Integer> colIndexes = PartitionUtil.getRefBaseTablePartitionColumIndexes(mv, refBaseTable);
if (colIndexes == null) {
return MVCompensation.createUnkownState(sessionVariable);
}
List<PartitionKey> newPartitionKeys = selectPartitionKeys.stream()
.map(partitionKey -> PartitionUtil.getSelectedPartitionKey(partitionKey, colIndexes))
.collect(Collectors.toList());
Set<String> selectPartitionNames = newPartitionKeys.stream()
Set<String> selectPartitionNames = selectPartitionKeys.stream()
.map(PartitionUtil::generateMVPartitionName)
.collect(Collectors.toSet());
if (selectPartitionNames.stream().noneMatch(refTablePartitionNamesToRefresh::contains)) {
Expand All @@ -362,7 +361,7 @@ private MVCompensation getMVCompensationForExternal(Table refBaseTable,
}
// if mv's to refresh partitions contains any of query's select partition ids, then rewrite with compensation.
List<PartitionKey> toRefreshRefTablePartitions = getMVCompensatePartitionsOfExternal(refBaseTable,
refTablePartitionNamesToRefresh, refScanOperator);
selectPartitionKeys, refTablePartitionNamesToRefresh, refScanOperator);
if (toRefreshRefTablePartitions == null) {
return MVCompensation.createUnkownState(sessionVariable);
}
Expand Down Expand Up @@ -407,37 +406,25 @@ private List<Long> getMVCompensatePartitionsOfOlap(Set<String> partitionNamesToR
}

private List<PartitionKey> getMVCompensatePartitionsOfExternal(Table refBaseTable,
List<PartitionKey> selectPartitionKeys,
Set<String> refTablePartitionNamesToRefresh,
LogicalScanOperator refScanOperator)
throws AnalysisException {
if (SUPPORTED_PARTITION_PRUNE_EXTERNAL_SCAN_TYPES.contains(refScanOperator.getOpType())) {
// For external table which support partition prune with OptExternalPartitionPruner,
// could use selectPartitionKeys to get the compensate partitions.
return getMVCompensatePartitionsOfExternalWithPartitionPruner(refTablePartitionNamesToRefresh,
refScanOperator);
return getMVCompensatePartitionsOfExternalWithPartitionPruner(selectPartitionKeys,
refTablePartitionNamesToRefresh, refScanOperator);
} else {
return getMVCompensatePartitionsOfExternalWithoutPartitionPruner(refBaseTable, refTablePartitionNamesToRefresh);
}
}

private List<PartitionKey> getMVCompensatePartitionsOfExternalWithPartitionPruner(
List<PartitionKey> selectPartitionKeys,
Set<String> refTablePartitionNamesToRefresh,
LogicalScanOperator refScanOperator) {
List<PartitionKey> refTableCompensatePartitionKeys = Lists.newArrayList();
ScanOperatorPredicates scanOperatorPredicates = null;
try {
scanOperatorPredicates = refScanOperator.getScanOperatorPredicates();
} catch (Exception e) {
return null;
}
if (scanOperatorPredicates == null) {
return null;
}
List<PartitionKey> selectPartitionKeys = scanOperatorPredicates.getSelectedPartitionKeys();
// different behavior for different external table types
if (selectPartitionKeys.isEmpty() && refScanOperator.getOpType() != OperatorType.LOGICAL_HIVE_SCAN) {
return null;
}
for (PartitionKey partitionKey : selectPartitionKeys) {
String partitionName = generateMVPartitionName(partitionKey);
if (refTablePartitionNamesToRefresh.contains(partitionName)) {
Expand Down
Loading