diff --git a/kafka-connect/kafka-connect-runtime/docker/docker-compose.yml b/kafka-connect/kafka-connect-runtime/docker/docker-compose.yml index cc9cd77ead0a..202180289d96 100644 --- a/kafka-connect/kafka-connect-runtime/docker/docker-compose.yml +++ b/kafka-connect/kafka-connect-runtime/docker/docker-compose.yml @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +volumes: + data: {} + services: minio: image: minio/minio @@ -22,16 +25,25 @@ services: environment: - MINIO_ROOT_USER=minioadmin - MINIO_ROOT_PASSWORD=minioadmin - - MINIO_DOMAIN=minio ports: - - 9001:9001 - 9000:9000 - command: ["server", "/data", "--console-address", ":9001"] + - 9001:9001 + volumes: + - data:/data + command: server /data --console-address ":9001" + + create-bucket: + image: minio/mc + depends_on: + - minio + volumes: + - data:/data + entrypoint: mc mb /data/bucket iceberg: image: tabulario/iceberg-rest depends_on: - - minio + - create-bucket hostname: iceberg ports: - 8181:8181 diff --git a/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationTestBase.java b/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationTestBase.java index 7b17cc8a6439..f90d4da0379e 100644 --- a/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationTestBase.java +++ b/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationTestBase.java @@ -41,12 +41,10 @@ import org.assertj.core.api.Condition; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import software.amazon.awssdk.services.s3.S3Client; public class IntegrationTestBase { private final TestContext context = TestContext.INSTANCE; - private S3Client s3; private Catalog catalog; private Admin admin; private String connectorName; @@ -60,18 +58,10 @@ protected TestContext context() { return context; } - protected S3Client s3() { - return s3; - } - protected Catalog catalog() { return catalog; } - protected Admin admin() { - return admin; - } - protected String connectorName() { return connectorName; } @@ -82,7 +72,6 @@ protected String testTopic() { @BeforeEach public void baseBefore() { - s3 = context.initLocalS3Client(); catalog = context.initLocalCatalog(); producer = context.initLocalProducer(); admin = context.initLocalAdmin(); @@ -102,7 +91,6 @@ public void baseAfter() { } producer.close(); admin.close(); - s3.close(); } protected void assertSnapshotProps(TableIdentifier tableIdentifier, String branch) { diff --git a/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/TestContext.java b/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/TestContext.java index d65ea19317fb..729d4bb264e5 100644 --- a/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/TestContext.java +++ b/kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/TestContext.java @@ -20,8 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Map; import java.util.UUID; import org.apache.iceberg.CatalogProperties; @@ -36,10 +34,6 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.testcontainers.containers.ComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; public class TestContext { @@ -50,7 +44,6 @@ public class TestContext { private static final int MINIO_PORT = 9000; private static final int CATALOG_PORT = 8181; private static final String BOOTSTRAP_SERVERS = "localhost:29092"; - private static final String BUCKET = "bucket"; private static final String AWS_ACCESS_KEY = "minioadmin"; private static final String AWS_SECRET_KEY = "minioadmin"; private static final String AWS_REGION = "us-east-1"; @@ -60,10 +53,6 @@ private TestContext() { new ComposeContainer(new File("./docker/docker-compose.yml")) .waitingFor("connect", Wait.forHttp("/connectors")); container.start(); - - try (S3Client s3 = initLocalS3Client()) { - s3.createBucket(req -> req.bucket(BUCKET)); - } } public void startConnector(KafkaConnectUtils.Config config) { @@ -75,21 +64,6 @@ public void stopConnector(String name) { KafkaConnectUtils.stopConnector(name); } - public S3Client initLocalS3Client() { - try { - return S3Client.builder() - .endpointOverride(new URI("http://localhost:" + MINIO_PORT)) - .region(Region.of(AWS_REGION)) - .forcePathStyle(true) - .credentialsProvider( - StaticCredentialsProvider.create( - AwsBasicCredentials.create(AWS_ACCESS_KEY, AWS_SECRET_KEY))) - .build(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - public Catalog initLocalCatalog() { String localCatalogUri = "http://localhost:" + CATALOG_PORT; RESTCatalog result = new RESTCatalog();