diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java new file mode 100644 index 0000000000..55f064b33a --- /dev/null +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.fs.gcs; + +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class GoogleHadoopFileSystemHTTPClientIntegrationTest + extends GoogleHadoopFileSystemIntegrationTest { + @Before + public void before() throws Exception { + storageClientType = ClientType.HTTP_API_CLIENT; + super.before(); + } +} diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java index e042757f3f..29b0f25824 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java @@ -62,12 +62,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; /** Integration tests for GoogleHadoopFileSystem class. */ -@RunWith(Parameterized.class) -public class GoogleHadoopFileSystemIntegrationTest extends GoogleHadoopFileSystemTestBase { +public abstract class GoogleHadoopFileSystemIntegrationTest extends GoogleHadoopFileSystemTestBase { private static final String PUBLIC_BUCKET = "gs://gcp-public-data-landsat"; diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java new file mode 100644 index 0000000000..c6aa525127 --- /dev/null +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java @@ -0,0 +1,64 @@ +package com.google.cloud.hadoop.fs.gcs; +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +// TODO: Ignored test for gRPC-directpath +// https://github.com/GoogleCloudDataproc/hadoop-connectors/issues/998 +@RunWith(JUnit4.class) +public class GoogleHadoopFileSystemJavaStorageClientIntegrationTest + extends GoogleHadoopFileSystemIntegrationTest { + + @Before + public void before() throws Exception { + storageClientType = ClientType.STORAGE_CLIENT; + super.before(); + } + + @Ignore + @Test + public void testImpersonationGroupNameIdentifierUsed() {} + + @Ignore + @Test + public void testImpersonationServiceAccountAndUserAndGroupNameIdentifierUsed() {} + + @Ignore + @Test + public void testImpersonationServiceAccountUsed() {} + + @Ignore + @Test + public void testImpersonationUserAndGroupNameIdentifiersUsed() {} + + @Ignore + @Test + public void testImpersonationUserNameIdentifierUsed() {} + + @Ignore + @Test + public void unauthenticatedAccessToPublicBuckets_fsGsProperties() {} + + @Ignore + @Test + public void unauthenticatedAccessToPublicBuckets_googleCloudProperties() {} +} diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java new file mode 100644 index 0000000000..56ae3cf7d9 --- /dev/null +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.gcsio; + +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class GoogleCloudStorageFileSystemHTTPClientTest + extends GoogleCloudStorageFileSystemTestBase { + + @Before + public void before() throws Exception { + storageClientType = ClientType.HTTP_API_CLIENT; + super.before(); + } +} diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java index 91f53af510..3e4d62e1ae 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java @@ -423,7 +423,8 @@ public void testGetAndListFileInfo() throws Exception { // At o1. validateListFileInfo(testBucket, "o1", /* expectedToExist= */ true, "o1"); if (getClass().equals(GoogleCloudStorageFileSystemIntegrationTest.class) - || getClass().equals(GoogleCloudStorageFileSystemTest.class)) { + || getClass().equals(GoogleCloudStorageFileSystemHTTPClientTest.class) + || getClass().equals(GoogleCloudStorageFileSystemJavaStorageClientTest.class)) { validateListFileInfo(testBucket, "o1/", /* expectedToExist= */ false); } else { validateListFileInfo(testBucket, "o1/", /* expectedToExist= */ true, "o1"); @@ -438,7 +439,8 @@ public void testGetAndListFileInfo() throws Exception { // At d1/o12. validateListFileInfo(testBucket, "d1/o12", /* expectedToExist= */ true, "d1/o12"); if (getClass().equals(GoogleCloudStorageFileSystemIntegrationTest.class) - || getClass().equals(GoogleCloudStorageFileSystemTest.class)) { + || getClass().equals(GoogleCloudStorageFileSystemHTTPClientTest.class) + || getClass().equals(GoogleCloudStorageFileSystemJavaStorageClientTest.class)) { validateListFileInfo(testBucket, "d1/o12/", /* expectedToExist= */ false); } else { validateListFileInfo(testBucket, "d1/o12/", /* expectedToExist= */ true, "d1/o12"); diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java new file mode 100644 index 0000000000..0d632da094 --- /dev/null +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.gcsio; + +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; +import java.io.IOException; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class GoogleCloudStorageFileSystemJavaStorageClientTest + extends GoogleCloudStorageFileSystemTestBase { + + @Before + public void before() throws Exception { + storageClientType = ClientType.STORAGE_CLIENT; + super.before(); + } + + @Override + @Ignore("DirectPath is not supported with null credentials") + @Test + public void testConstructor() throws IOException {} + + @Override + @Ignore("DirectPath is not supported with null credentials") + @Test + public void testClientType() {} +} diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTestBase.java similarity index 98% rename from gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTest.java rename to gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTestBase.java index 36569d8fbf..ccd6d7dc77 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTestBase.java @@ -33,16 +33,14 @@ import java.util.logging.Logger; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; /** * The unittest version of {@code GoogleCloudStorageFileSystemIntegrationTest}; the external * GoogleCloudStorage dependency is replaced by an in-memory version which mimics the same * bucket/object semantics. */ -@RunWith(Parameterized.class) -public class GoogleCloudStorageFileSystemTest extends GoogleCloudStorageFileSystemIntegrationTest { +public abstract class GoogleCloudStorageFileSystemTestBase + extends GoogleCloudStorageFileSystemIntegrationTest { @Before public void before() throws Exception { diff --git a/pom.xml b/pom.xml index 0765ccb67f..0484a71807 100644 --- a/pom.xml +++ b/pom.xml @@ -91,9 +91,9 @@ v1-rev20220705-${google.api-client-libraries.version} 1.12.1 1.10.1 - 2.28.4 + 2.36.0 2.5.4 - 2.17.2 + 2.22.1 0.7.1 2.19.5 2.8.9 @@ -101,7 +101,7 @@ 1.42.3 1.34.1 3.21.9 - 1.52.1 + 1.54.0 2.10.2 3.2.4 0.31.0 diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh index c2d23a1d9a..ad14c80ae9 100755 --- a/tools/run_integration_tests.sh +++ b/tools/run_integration_tests.sh @@ -75,5 +75,7 @@ fi export GCS_TEST_PRIVATE_KEYFILE export HDFS_ROOT=file:///tmp export RUN_INTEGRATION_TESTS=true +# Env variable to enable direct path over gRPC +export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true ./mvnw -B -e -T1C "-P${HADOOP_VERSION}" -Pintegration-test clean verify "${@:5}"