[core] AppendOnlySplitGenerator skips the check of sequence number ov…

…erlap in UNAWARE mode to reduce invalid logs.
liming30 · Nov 27, 2023 · aac3d40 · aac3d40
1 parent 385bd90
commit aac3d40
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyCompactManager.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyCompactManager.java
@@ -70,7 +70,7 @@ public AppendOnlyCompactManager(
             CompactRewriter rewriter,
             @Nullable CompactionMetrics metrics) {
         this.executor = executor;
-        this.toCompact = new TreeSet<>(fileComparator());
+        this.toCompact = new TreeSet<>(fileComparator(false));
         this.toCompact.addAll(restored);
         this.minFileNum = minFileNum;
         this.maxFileNum = maxFileNum;
@@ -304,13 +304,13 @@ public interface CompactRewriter {
      * may be put after the new files, and this order will be disrupted. We need to ensure this
      * order, so we force the order by sequence.
      */
-    public static Comparator<DataFileMeta> fileComparator() {
+    public static Comparator<DataFileMeta> fileComparator(boolean ignoreOverlap) {
         return (o1, o2) -> {
             if (o1 == o2) {
                 return 0;
             }
 
-            if (isOverlap(o1, o2)) {
+            if (!ignoreOverlap && isOverlap(o1, o2)) {
                 LOG.warn(
                         String.format(
                                 "There should no overlap in append files, but Range1(%s, %s), Range2(%s, %s),"

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java
@@ -46,7 +46,7 @@ public AppendOnlySplitGenerator(
     @Override
     public List<List<DataFileMeta>> splitForBatch(List<DataFileMeta> input) {
         List<DataFileMeta> files = new ArrayList<>(input);
-        files.sort(fileComparator());
+        files.sort(fileComparator(bucketMode == BucketMode.UNAWARE));
         Function<DataFileMeta, Long> weightFunc = file -> Math.max(file.fileSize(), openFileCost);
         return BinPacking.packForOrdered(files, weightFunc, targetSplitSize);
     }