Merge branch 'main' into kafka-connect-pt1

apache · Dec 6, 2023 · f6e7bbd · f6e7bbd
2 parents 9fed273 + 70ec4e5
commit f6e7bbd
Show file tree

Hide file tree

Showing 320 changed files with 14,767 additions and 1,845 deletions.
diff --git a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml
@@ -9,7 +9,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "1.4.1 (latest release)"
+        - "1.4.2 (latest release)"
+        - "1.4.1"
         - "1.4.0"
         - "1.3.1"
         - "1.3.0"

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -30,4 +30,10 @@ updates:
       interval: "weekly"
       day: "sunday"
     open-pull-requests-limit: 50
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    open-pull-requests-limit: 5
 
diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml
@@ -46,7 +46,7 @@ jobs:
           #
           # See https://github.com/actions/checkout/issues/124
           fetch-depth: 0
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: 11

diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml
@@ -62,7 +62,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ matrix.jvm }}
@@ -91,7 +91,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ matrix.jvm }}

diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml
@@ -65,7 +65,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ matrix.jvm }}

diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml
@@ -60,7 +60,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ matrix.jvm }}
@@ -86,7 +86,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: 8

diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml
@@ -57,7 +57,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ matrix.jvm }}
@@ -81,7 +81,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: 8
@@ -91,7 +91,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: 8

diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml
@@ -79,7 +79,7 @@ jobs:
       with:
         repository: ${{ github.event.inputs.repo }}
         ref: ${{ github.event.inputs.ref }}
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: 11

diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
@@ -34,7 +34,7 @@ jobs:
         with:
           # we need to fetch all tags so that getProjectVersion() in build.gradle correctly determines the next SNAPSHOT version from the newest tag
           fetch-depth: 0
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: 8

diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml
@@ -49,7 +49,7 @@ jobs:
         with:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.ref }}
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: 11

diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml
@@ -63,7 +63,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ matrix.jvm }}
@@ -93,7 +93,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ matrix.jvm }}
@@ -123,7 +123,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: 17

diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml
@@ -872,6 +872,11 @@ acceptedBreaks:
       old: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
       new: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
       justification: "Static utility class - should not have public constructor"
+  "1.4.0":
+    org.apache.iceberg:iceberg-core:
+    - code: "java.field.serialVersionUIDChanged"
+      new: "field org.apache.iceberg.util.SerializableMap<K, V>.serialVersionUID"
+      justification: "Serialization is not be used"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/TestAliyunClientFactories.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/TestAliyunClientFactories.java
@@ -22,43 +22,44 @@
 import java.util.Map;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.junit.Assert;
-import org.junit.Test;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
 
 public class TestAliyunClientFactories {
 
   @Test
   public void testLoadDefault() {
-    Assert.assertEquals(
-        "Default client should be singleton",
-        AliyunClientFactories.defaultFactory(),
-        AliyunClientFactories.defaultFactory());
+    Assertions.assertThat(AliyunClientFactories.defaultFactory())
+        .as("Default client should be singleton")
+        .isEqualTo(AliyunClientFactories.defaultFactory());
 
     AliyunClientFactory defaultFactory = AliyunClientFactories.from(Maps.newHashMap());
-    Assert.assertTrue(
-        "Should load default when factory impl not configured",
-        defaultFactory instanceof AliyunClientFactories.DefaultAliyunClientFactory);
-    Assert.assertNull(
-        "Should have no Aliyun properties set", defaultFactory.aliyunProperties().accessKeyId());
+    Assertions.assertThat(defaultFactory)
+        .as("Should load default when factory impl not configured")
+        .isInstanceOf(AliyunClientFactories.DefaultAliyunClientFactory.class);
+
+    Assertions.assertThat(defaultFactory.aliyunProperties().accessKeyId())
+        .as("Should have no Aliyun properties set")
+        .isNull();
 
     AliyunClientFactory defaultFactoryWithConfig =
         AliyunClientFactories.from(ImmutableMap.of(AliyunProperties.CLIENT_ACCESS_KEY_ID, "key"));
-    Assert.assertTrue(
-        "Should load default when factory impl not configured",
-        defaultFactoryWithConfig instanceof AliyunClientFactories.DefaultAliyunClientFactory);
-    Assert.assertEquals(
-        "Should have access key set",
-        "key",
-        defaultFactoryWithConfig.aliyunProperties().accessKeyId());
+    Assertions.assertThat(defaultFactoryWithConfig)
+        .as("Should load default when factory impl not configured")
+        .isInstanceOf(AliyunClientFactories.DefaultAliyunClientFactory.class);
+
+    Assertions.assertThat(defaultFactoryWithConfig.aliyunProperties().accessKeyId())
+        .as("Should have access key set")
+        .isEqualTo("key");
   }
 
   @Test
   public void testLoadCustom() {
     Map<String, String> properties = Maps.newHashMap();
     properties.put(AliyunProperties.CLIENT_FACTORY, CustomFactory.class.getName());
-    Assert.assertTrue(
-        "Should load custom class",
-        AliyunClientFactories.from(properties) instanceof CustomFactory);
+    Assertions.assertThat(AliyunClientFactories.from(properties))
+        .as("Should load custom class")
+        .isInstanceOf(CustomFactory.class);
   }
 
   public static class CustomFactory implements AliyunClientFactory {

diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java
@@ -18,9 +18,9 @@
  */
 package org.apache.iceberg.aliyun;
 
-import org.apache.iceberg.aliyun.oss.AliyunOSSTestRule;
+import org.apache.iceberg.aliyun.oss.AliyunOSSExtension;
 import org.apache.iceberg.aliyun.oss.OSSURI;
-import org.apache.iceberg.aliyun.oss.mock.AliyunOSSMockRule;
+import org.apache.iceberg.aliyun.oss.mock.AliyunOSSMockExtension;
 import org.apache.iceberg.common.DynConstructors;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.base.Strings;
@@ -41,33 +41,34 @@ public class TestUtility {
 
   private TestUtility() {}
 
-  public static AliyunOSSTestRule initialize() {
-    AliyunOSSTestRule testRule;
+  public static AliyunOSSExtension initialize() {
+    AliyunOSSExtension extension;
 
     String implClass = System.getenv(ALIYUN_TEST_OSS_RULE_CLASS);
     if (!Strings.isNullOrEmpty(implClass)) {
-      LOG.info("The initializing AliyunOSSTestRule implementation is: {}", implClass);
+      LOG.info("The initializing AliyunOSSExtension implementation is: {}", implClass);
       try {
-        DynConstructors.Ctor<AliyunOSSTestRule> ctor =
-            DynConstructors.builder(AliyunOSSTestRule.class).impl(implClass).buildChecked();
-        testRule = ctor.newInstance();
+        DynConstructors.Ctor<AliyunOSSExtension> ctor =
+            DynConstructors.builder(AliyunOSSExtension.class).impl(implClass).buildChecked();
+        extension = ctor.newInstance();
       } catch (NoSuchMethodException e) {
         throw new IllegalArgumentException(
             String.format(
-                "Cannot initialize AliyunOSSTestRule, missing no-arg constructor: %s", implClass),
+                "Cannot initialize AliyunOSSExtension, missing no-arg constructor: %s", implClass),
             e);
       } catch (ClassCastException e) {
         throw new IllegalArgumentException(
             String.format(
-                "Cannot initialize AliyunOSSTestRule, %s does not implement it.", implClass),
+                "Cannot initialize AliyunOSSExtension, %s does not implement it.", implClass),
             e);
       }
     } else {
-      LOG.info("Initializing AliyunOSSTestRule implementation with default AliyunOSSMockRule");
-      testRule = AliyunOSSMockRule.builder().silent().build();
+      LOG.info(
+          "Initializing AliyunOSSExtension implementation with default AliyunOSSMockExtension");
+      extension = AliyunOSSMockExtension.builder().silent().build();
     }
 
-    return testRule;
+    return extension;
   }
 
   public static String accessKeyId() {

diff --git a/...iceberg/aliyun/oss/AliyunOSSTestRule.java → ...ceberg/aliyun/oss/AliyunOSSExtension.java b/...iceberg/aliyun/oss/AliyunOSSTestRule.java → ...ceberg/aliyun/oss/AliyunOSSExtension.java
@@ -20,17 +20,17 @@
 
 import com.aliyun.oss.OSS;
 import java.util.UUID;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
 
 /**
  * API for test Aliyun Object Storage Service (OSS) which is either local mock http server or remote
  * aliyun oss server
  *
  * <p>This API includes start,stop OSS service, create OSS client, setup bucket and teardown bucket.
  */
-public interface AliyunOSSTestRule extends TestRule {
+public interface AliyunOSSExtension extends BeforeAllCallback, AfterAllCallback {
   UUID RANDOM_UUID = java.util.UUID.randomUUID();
 
   /** Returns a specific bucket name for testing purpose. */
@@ -39,18 +39,13 @@ default String testBucketName() {
   }
 
   @Override
-  default Statement apply(Statement base, Description description) {
-    return new Statement() {
-      @Override
-      public void evaluate() throws Throwable {
-        start();
-        try {
-          base.evaluate();
-        } finally {
-          stop();
-        }
-      }
-    };
+  default void afterAll(ExtensionContext context) throws Exception {
+    stop();
+  }
+
+  @Override
+  default void beforeAll(ExtensionContext context) throws Exception {
+    start();
   }
 
   /**

diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java
@@ -21,25 +21,26 @@
 import com.aliyun.oss.OSS;
 import org.apache.iceberg.aliyun.TestUtility;
 import org.apache.iceberg.util.SerializableSupplier;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 public abstract class AliyunOSSTestBase {
-  @ClassRule public static final AliyunOSSTestRule OSS_TEST_RULE = TestUtility.initialize();
+  @RegisterExtension
+  private static final AliyunOSSExtension OSS_TEST_EXTENSION = TestUtility.initialize();
 
-  private final SerializableSupplier<OSS> ossClient = OSS_TEST_RULE::createOSSClient;
-  private final String bucketName = OSS_TEST_RULE.testBucketName();
-  private final String keyPrefix = OSS_TEST_RULE.keyPrefix();
+  private final SerializableSupplier<OSS> ossClient = OSS_TEST_EXTENSION::createOSSClient;
+  private final String bucketName = OSS_TEST_EXTENSION.testBucketName();
+  private final String keyPrefix = OSS_TEST_EXTENSION.keyPrefix();
 
-  @Before
+  @BeforeEach
   public void before() {
-    OSS_TEST_RULE.setUpBucket(bucketName);
+    OSS_TEST_EXTENSION.setUpBucket(bucketName);
   }
 
-  @After
+  @AfterEach
   public void after() {
-    OSS_TEST_RULE.tearDownBucket(bucketName);
+    OSS_TEST_EXTENSION.tearDownBucket(bucketName);
   }
 
   protected String location(String key) {

diff --git a/...rg/aliyun/oss/OSSIntegrationTestRule.java → ...g/aliyun/oss/OSSIntegrationExtension.java b/...rg/aliyun/oss/OSSIntegrationTestRule.java → ...g/aliyun/oss/OSSIntegrationExtension.java
@@ -26,7 +26,7 @@
 import org.apache.iceberg.aliyun.TestUtility;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 
-public class OSSIntegrationTestRule implements AliyunOSSTestRule {
+public class OSSIntegrationExtension implements AliyunOSSExtension {
   // Aliyun access key pair.
   private String accessKeyId;
   private String accessKeySecret;
@@ -106,7 +106,7 @@ public void tearDownBucket(String bucket) {
 
   private OSS ossClient() {
     if (lazyClient == null) {
-      synchronized (OSSIntegrationTestRule.class) {
+      synchronized (OSSIntegrationExtension.class) {
         if (lazyClient == null) {
           lazyClient = createOSSClient();
         }