Skip to content

Commit

Permalink
give some examples using TestBase
Browse files Browse the repository at this point in the history
  • Loading branch information
@lisirrx
lisirrx committed Jan 4, 2024
1 parent 32cc331 commit d695238
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 116 deletions.
6 changes: 3 additions & 3 deletions core/src/test/java/org/apache/iceberg/TestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ ManifestFile writeManifest(DataFile... files) throws IOException {

ManifestFile writeManifest(Long snapshotId, DataFile... files) throws IOException {
File manifestFile = temp.resolve("input.m0.avro").toFile();
Assertions.assertThat(manifestFile.delete()).isTrue();
Assertions.assertThat(manifestFile).doesNotExist();
OutputFile outputFile = table.ops().io().newOutputFile(manifestFile.getCanonicalPath());

ManifestWriter<DataFile> writer =
Expand Down Expand Up @@ -264,7 +264,7 @@ ManifestFile writeManifest(Long snapshotId, ManifestEntry<?>... entries) throws
<F extends ContentFile<F>> ManifestFile writeManifest(
Long snapshotId, String fileName, ManifestEntry<?>... entries) throws IOException {
File manifestFile = temp.resolve(fileName).toFile();
Assertions.assertThat(manifestFile.delete()).isTrue();
Assertions.assertThat(manifestFile).doesNotExist();
OutputFile outputFile = table.ops().io().newOutputFile(manifestFile.getCanonicalPath());

ManifestWriter<F> writer;
Expand Down Expand Up @@ -309,7 +309,7 @@ ManifestFile writeDeleteManifest(int newFormatVersion, Long snapshotId, DeleteFi

ManifestFile writeManifestWithName(String name, DataFile... files) throws IOException {
File manifestFile = temp.resolve(name + ".avro").toFile();
Assertions.assertThat(manifestFile.delete()).isTrue();
Assertions.assertThat(manifestFile).doesNotExist();
OutputFile outputFile = table.ops().io().newOutputFile(manifestFile.getCanonicalPath());

ManifestWriter<DataFile> writer =
Expand Down
135 changes: 68 additions & 67 deletions core/src/test/java/org/apache/iceberg/TestCreateSnapshotEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,20 @@
*/
package org.apache.iceberg;

import java.util.Arrays;
import java.util.Collection;
import org.apache.iceberg.events.CreateSnapshotEvent;
import org.apache.iceberg.events.Listener;
import org.apache.iceberg.events.Listeners;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;

@RunWith(Parameterized.class)
public class TestCreateSnapshotEvent extends TableTestBase {
@Parameterized.Parameters(name = "formatVersion = {0}")
public static Object[] parameters() {
return new Object[] {1, 2};
@ExtendWith(ParameterizedTestExtension.class)
public class TestCreateSnapshotEvent extends TestBase {
@Parameters(name = "formatVersion={0}")
public static Collection<Integer> parameters() {
return Arrays.asList(1, 2);
}

private CreateSnapshotEvent currentEvent;
Expand All @@ -40,76 +41,76 @@ public TestCreateSnapshotEvent(int formatVersion) {
Listeners.register(new MyListener(), CreateSnapshotEvent.class);
}

@Test
@TestTemplate
public void testAppendCommitEvent() {
Assert.assertEquals("Table should start empty", 0, listManifestFiles().size());
Assertions.assertThat(listManifestFiles()).as("Table should start empty").isEmpty();

table.newAppend().appendFile(FILE_A).commit();
Assert.assertNotNull(currentEvent);
Assert.assertEquals(
"Added records in the table should be 1", "1", currentEvent.summary().get("added-records"));
Assert.assertEquals(
"Added files in the table should be 1",
"1",
currentEvent.summary().get("added-data-files"));
Assert.assertEquals(
"Total records in the table should be 1", "1", currentEvent.summary().get("total-records"));
Assert.assertEquals(
"Total data files in the table should be 1",
"1",
currentEvent.summary().get("total-data-files"));
Assertions.assertThat(currentEvent).isNotNull();
Assertions.assertThat(currentEvent.summary().get("added-records"))
.as("Added records in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("added-data-files"))
.as("Added files in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("total-records"))
.as("Total records in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("total-data-files"))
.as("Total data files in the table should be 1")
.isEqualTo("1");

table.newAppend().appendFile(FILE_A).commit();
Assert.assertNotNull(currentEvent);
Assert.assertEquals(
"Added records in the table should be 1", "1", currentEvent.summary().get("added-records"));
Assert.assertEquals(
"Added files in the table should be 1",
"1",
currentEvent.summary().get("added-data-files"));
Assert.assertEquals(
"Total records in the table should be 2", "2", currentEvent.summary().get("total-records"));
Assert.assertEquals(
"Total data files in the table should be 2",
"2",
currentEvent.summary().get("total-data-files"));
Assertions.assertThat(currentEvent).isNotNull();
Assertions.assertThat(currentEvent.summary().get("added-records"))
.as("Added records in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("added-data-files"))
.as("Added files in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("total-records"))
.as("Total records in the table should be 2")
.isEqualTo("2");
Assertions.assertThat(currentEvent.summary().get("total-data-files"))
.as("Total data files in the table should be 2")
.isEqualTo("2");
}

@Test
@TestTemplate
public void testAppendAndDeleteCommitEvent() {
Assert.assertEquals("Table should start empty", 0, listManifestFiles().size());
Assertions.assertThat(listManifestFiles()).as("Table should start empty").isEmpty();

table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();
Assert.assertNotNull(currentEvent);
Assert.assertEquals(
"Added records in the table should be 2", "2", currentEvent.summary().get("added-records"));
Assert.assertEquals(
"Added files in the table should be 2",
"2",
currentEvent.summary().get("added-data-files"));
Assert.assertEquals(
"Total records in the table should be 2", "2", currentEvent.summary().get("total-records"));
Assert.assertEquals(
"Total data files in the table should be 2",
"2",
currentEvent.summary().get("total-data-files"));
Assertions.assertThat(currentEvent).as("Current event should not be null").isNotNull();
Assertions.assertThat(currentEvent.summary().get("added-records"))
.as("Added records in the table should be 2")
.isEqualTo("2");
Assertions.assertThat(currentEvent.summary().get("added-data-files"))
.as("Added files in the table should be 2")
.isEqualTo("2");
Assertions.assertThat(currentEvent.summary().get("total-records"))
.as("Total records in the table should be 2")
.isEqualTo("2");
Assertions.assertThat(currentEvent.summary().get("total-data-files"))
.as("Total data files in the table should be 2")
.isEqualTo("2");

table.newDelete().deleteFile(FILE_A).commit();
Assert.assertNotNull(currentEvent);
Assert.assertEquals(
"Deleted records in the table should be 1",
"1",
currentEvent.summary().get("deleted-records"));
Assert.assertEquals(
"Deleted files in the table should be 1",
"1",
currentEvent.summary().get("deleted-data-files"));
Assert.assertEquals(
"Total records in the table should be 1", "1", currentEvent.summary().get("total-records"));
Assert.assertEquals(
"Total data files in the table should be 1",
"1",
currentEvent.summary().get("total-data-files"));
Assertions.assertThat(currentEvent)
.as("Current event should not be null after delete")
.isNotNull();
Assertions.assertThat(currentEvent.summary().get("deleted-records"))
.as("Deleted records in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("deleted-data-files"))
.as("Deleted files in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("total-records"))
.as("Total records in the table should be 1")
.isEqualTo("1");
Assertions.assertThat(currentEvent.summary().get("total-data-files"))
.as("Total data files in the table should be 1")
.isEqualTo("1");
}

class MyListener implements Listener<CreateSnapshotEvent> {
Expand Down
93 changes: 47 additions & 46 deletions core/src/test/java/org/apache/iceberg/TestManifestReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
*/
package org.apache.iceberg;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.iceberg.ManifestEntry.Status;
Expand All @@ -32,17 +32,15 @@
import org.apache.iceberg.types.Types;
import org.assertj.core.api.Assertions;
import org.assertj.core.api.recursive.comparison.RecursiveComparisonConfiguration;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

@RunWith(Parameterized.class)
public class TestManifestReader extends TableTestBase {
@Parameterized.Parameters(name = "formatVersion = {0}")
public static Object[] parameters() {
return new Object[] {1, 2};
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;

@ExtendWith(ParameterizedTestExtension.class)
public class TestManifestReader extends TestBase {
@Parameters(name = "formatVersion = {0}")
public static Collection<Integer> parameters() {
return Arrays.asList(1, 2);
}

private static final RecursiveComparisonConfiguration FILE_COMPARISON_CONFIG =
Expand All @@ -55,18 +53,18 @@ public TestManifestReader(int formatVersion) {
super(formatVersion);
}

@Test
@TestTemplate
public void testManifestReaderWithEmptyInheritableMetadata() throws IOException {
ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 1000L, FILE_A));
try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
Assert.assertEquals(Status.EXISTING, entry.status());
Assert.assertEquals(FILE_A.path(), entry.file().path());
Assert.assertEquals(1000L, (long) entry.snapshotId());
Assertions.assertThat(entry.status()).isEqualTo(Status.EXISTING);
Assertions.assertThat(entry.file().path()).isEqualTo(FILE_A.path());
Assertions.assertThat(entry.snapshotId()).isEqualTo(1000L);
}
}

@Test
@TestTemplate
public void testReaderWithFilterWithoutSelect() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE_A, FILE_B, FILE_C);
try (ManifestReader<DataFile> reader =
Expand All @@ -76,37 +74,38 @@ public void testReaderWithFilterWithoutSelect() throws IOException {
// note that all files are returned because the reader returns data files that may match, and
// the partition is
// bucketing by data, which doesn't help filter files
assertThat(files)
Assertions.assertThat(files)
.usingRecursiveComparison(FILE_COMPARISON_CONFIG)
.isEqualTo(Lists.newArrayList(FILE_A, FILE_B, FILE_C));
}
}

@Test
@TestTemplate
public void testInvalidUsage() throws IOException {
ManifestFile manifest = writeManifest(FILE_A, FILE_B);
Assertions.assertThatThrownBy(() -> ManifestFiles.read(manifest, FILE_IO))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot read from ManifestFile with null (unassigned) snapshot ID");
}

@Test
@TestTemplate
public void testManifestReaderWithPartitionMetadata() throws IOException {
ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 123L, FILE_A));
try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
Assert.assertEquals(123L, (long) entry.snapshotId());
Assertions.assertThat(entry.snapshotId()).isEqualTo(123L);

List<Types.NestedField> fields =
((PartitionData) entry.file().partition()).getPartitionType().fields();
Assert.assertEquals(1, fields.size());
Assert.assertEquals(1000, fields.get(0).fieldId());
Assert.assertEquals("data_bucket", fields.get(0).name());
Assert.assertEquals(Types.IntegerType.get(), fields.get(0).type());

Assertions.assertThat(fields).hasSize(1);
Assertions.assertThat(fields.get(0).fieldId()).isEqualTo(1000);
Assertions.assertThat(fields.get(0).name()).isEqualTo("data_bucket");
Assertions.assertThat(fields.get(0).type()).isEqualTo(Types.IntegerType.get());
}
}

@Test
@TestTemplate
public void testManifestReaderWithUpdatedPartitionMetadataForV1Table() throws IOException {
PartitionSpec spec =
PartitionSpec.builderFor(table.schema()).bucket("id", 8).bucket("data", 16).build();
Expand All @@ -115,53 +114,55 @@ public void testManifestReaderWithUpdatedPartitionMetadataForV1Table() throws IO
ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 123L, FILE_A));
try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
Assert.assertEquals(123L, (long) entry.snapshotId());
Assertions.assertThat(entry.snapshotId()).isEqualTo(123L);

List<Types.NestedField> fields =
((PartitionData) entry.file().partition()).getPartitionType().fields();
Assert.assertEquals(2, fields.size());
Assert.assertEquals(1000, fields.get(0).fieldId());
Assert.assertEquals("id_bucket", fields.get(0).name());
Assert.assertEquals(Types.IntegerType.get(), fields.get(0).type());

Assert.assertEquals(1001, fields.get(1).fieldId());
Assert.assertEquals("data_bucket", fields.get(1).name());
Assert.assertEquals(Types.IntegerType.get(), fields.get(1).type());
Assertions.assertThat(fields).hasSize(2);
Assertions.assertThat(fields.get(0).fieldId()).isEqualTo(1000);
Assertions.assertThat(fields.get(0).name()).isEqualTo("id_bucket");
Assertions.assertThat(fields.get(0).type()).isEqualTo(Types.IntegerType.get());

Assertions.assertThat(fields.get(1).fieldId()).isEqualTo(1001);
Assertions.assertThat(fields.get(1).name()).isEqualTo("data_bucket");
Assertions.assertThat(fields.get(1).type()).isEqualTo(Types.IntegerType.get());
}
}

@Test
@TestTemplate
public void testDataFilePositions() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE_A, FILE_B, FILE_C);
try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
long expectedPos = 0L;
for (DataFile file : reader) {
Assert.assertEquals("Position should match", (Long) expectedPos, file.pos());
Assert.assertEquals(
"Position from field index should match", expectedPos, ((BaseFile) file).get(17));
Assertions.assertThat(file.pos()).as("Position should match").isEqualTo(expectedPos);
Assertions.assertThat(((BaseFile) file).get(17))
.as("Position from field index should match")
.isEqualTo(expectedPos);
expectedPos += 1;
}
}
}

@Test
@TestTemplate
public void testDeleteFilePositions() throws IOException {
Assume.assumeTrue("Delete files only work for format version 2", formatVersion == 2);
Assumptions.assumeTrue(formatVersion == 2, "Delete files only work for format version 2");
ManifestFile manifest =
writeDeleteManifest(formatVersion, 1000L, FILE_A_DELETES, FILE_B_DELETES);
try (ManifestReader<DeleteFile> reader =
ManifestFiles.readDeleteManifest(manifest, FILE_IO, null)) {
long expectedPos = 0L;
for (DeleteFile file : reader) {
Assert.assertEquals("Position should match", (Long) expectedPos, file.pos());
Assert.assertEquals(
"Position from field index should match", expectedPos, ((BaseFile) file).get(17));
Assertions.assertThat(file.pos()).as("Position should match").isEqualTo(expectedPos);
Assertions.assertThat(((BaseFile) file).get(17))
.as("Position from field index should match")
.isEqualTo(expectedPos);
expectedPos += 1;
}
}
}

@Test
@TestTemplate
public void testDataFileSplitOffsetsNullWhenInvalid() throws IOException {
DataFile invalidOffset =
DataFiles.builder(SPEC)
Expand Down

0 comments on commit d695238

Please sign in to comment.