Skip to content

Commit

Permalink
Remove matches
Browse files Browse the repository at this point in the history
  • Loading branch information
@TQJADE
TQJADE committed Dec 19, 2024
1 parent 4774774 commit a242dfc
Showing 1 changed file with 31 additions and 30 deletions.
61 changes: 31 additions & 30 deletions .../src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
import org.apache.spark.sql.internal.SQLConf;
import org.apache.spark.sql.types.StructType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.io.TempDir;

Expand Down Expand Up @@ -202,8 +203,8 @@ public void testUnpartitioned() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -238,8 +239,8 @@ public void testRewriteAll() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -284,8 +285,8 @@ public void testRewriteFilter() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);

withSQLConf(
ImmutableMap.of(SQLConf.CASE_SENSITIVE().key(), "true"),
Expand Down Expand Up @@ -334,8 +335,8 @@ public void testRewriteToSmallerTarget() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -379,8 +380,8 @@ public void testRemoveDanglingDeletes() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertThat(actualDeletes).as("No new position deletes").isEmpty();
assertEquals("Rows", expectedRecords, actualRecords);
assertThat(actualDeletes).as("New position deletes").isEmpty();
}

@TestTemplate
Expand Down Expand Up @@ -432,8 +433,8 @@ public void testSomePartitionsDanglingDeletes() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -480,8 +481,8 @@ public void testRewriteFilterRemoveDangling() throws Exception {
// Only non-compacted deletes remain
List<Object[]> expectedDeletesFiltered =
filterDeletes(expectedDeletes, ImmutableList.of(2), ImmutableList.of(3));
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletesFiltered, allDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletesFiltered, allDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -532,8 +533,8 @@ public void testPartitionEvolutionAdd() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -579,8 +580,8 @@ public void testPartitionEvolutionRemove() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

@TestTemplate
Expand Down Expand Up @@ -629,7 +630,7 @@ public void testSchemaEvolution() throws Exception {
checkSequenceNumbers(table, rewrittenDeleteFiles, newDeleteFiles);

List<Object[]> actualRecords = records(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Rows", expectedRecords, actualRecords);
}

@TestTemplate
Expand Down Expand Up @@ -724,8 +725,8 @@ public void testRewriteManyColumns() throws Exception {

List<Object[]> actualRecords = records(table);
List<Object[]> actualDeletes = deleteRecords(table);
assertEquals("Rows must match", expectedRecords, actualRecords);
assertEquals("Position deletes must match", expectedDeletes, actualDeletes);
assertEquals("Rows", expectedRecords, actualRecords);
assertEquals("Position deletes", expectedDeletes, actualDeletes);
}

private Table createTablePartitioned(int partitions, int files, int numRecords) {
Expand Down Expand Up @@ -1054,48 +1055,48 @@ private void checkResult(
List<DeleteFile> newDeletes,
int expectedGroups) {
assertThat(rewrittenDeletes.size())
.as("Rewritten delete file count does not match")
.as("Rewritten delete file count")
.isEqualTo(result.rewrittenDeleteFilesCount());

assertThat(newDeletes.size())
.as("New delete file count does not match")
.as("New delete file count")
.isEqualTo(result.addedDeleteFilesCount());

assertThat(size(rewrittenDeletes))
.as("Rewritten delete byte count does not match")
.as("Rewritten delete byte count")
.isEqualTo(result.rewrittenBytesCount());

assertThat(size(newDeletes))
.as("New delete byte count does not match")
.as("New delete byte count")
.isEqualTo(result.addedBytesCount());

assertThat(expectedGroups)
.as("Rewrite group count does not match")
.as("Rewrite group count")
.isEqualTo(result.rewriteResults().size());

assertThat(rewrittenDeletes.size())
.as("Rewritten delete file count in all groups to match")
.as("Rewritten delete file count in all groups")
.isEqualTo(
result.rewriteResults().stream()
.mapToInt(FileGroupRewriteResult::rewrittenDeleteFilesCount)
.sum());

assertThat(newDeletes.size())
.as("Added delete file count in all groups to match")
.as("Added delete file count in all groups")
.isEqualTo(
result.rewriteResults().stream()
.mapToInt(FileGroupRewriteResult::addedDeleteFilesCount)
.sum());

assertThat(size(rewrittenDeletes))
.as("Rewritten delete bytes in all groups to match")
.as("Rewritten delete bytes in all groups")
.isEqualTo(
result.rewriteResults().stream()
.mapToLong(FileGroupRewriteResult::rewrittenBytesCount)
.sum());

assertThat(size(newDeletes))
.as("Added delete bytes in all groups to match")
.as("Added delete bytes in all groups")
.isEqualTo(
result.rewriteResults().stream()
.mapToLong(FileGroupRewriteResult::addedBytesCount)
Expand Down

0 comments on commit a242dfc

Please sign in to comment.