upgrade google-java-storage (#997)

GoogleCloudDataproc · May 8, 2023 · 267fda3 · 267fda3
1 parent 69eb118
commit 267fda3
Show file tree

Hide file tree

Showing 9 changed files with 187 additions and 13 deletions.
diff --git a/.../java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java b/.../java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.fs.gcs;
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GoogleHadoopFileSystemHTTPClientIntegrationTest
+    extends GoogleHadoopFileSystemIntegrationTest {
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.HTTP_API_CLIENT;
+    super.before();
+  }
+}
diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java
@@ -62,12 +62,9 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /** Integration tests for GoogleHadoopFileSystem class. */
-@RunWith(Parameterized.class)
-public class GoogleHadoopFileSystemIntegrationTest extends GoogleHadoopFileSystemTestBase {
+public abstract class GoogleHadoopFileSystemIntegrationTest extends GoogleHadoopFileSystemTestBase {
 
   private static final String PUBLIC_BUCKET = "gs://gcp-public-data-landsat";
 

diff --git a/...om/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java b/...om/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java
@@ -0,0 +1,64 @@
+package com.google.cloud.hadoop.fs.gcs;
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+// TODO: Ignored test for gRPC-directpath
+// https://github.com/GoogleCloudDataproc/hadoop-connectors/issues/998
+@RunWith(JUnit4.class)
+public class GoogleHadoopFileSystemJavaStorageClientIntegrationTest
+    extends GoogleHadoopFileSystemIntegrationTest {
+
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.STORAGE_CLIENT;
+    super.before();
+  }
+
+  @Ignore
+  @Test
+  public void testImpersonationGroupNameIdentifierUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationServiceAccountAndUserAndGroupNameIdentifierUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationServiceAccountUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationUserAndGroupNameIdentifiersUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationUserNameIdentifierUsed() {}
+
+  @Ignore
+  @Test
+  public void unauthenticatedAccessToPublicBuckets_fsGsProperties() {}
+
+  @Ignore
+  @Test
+  public void unauthenticatedAccessToPublicBuckets_googleCloudProperties() {}
+}
diff --git a/...c/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java b/...c/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.gcsio;
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GoogleCloudStorageFileSystemHTTPClientTest
+    extends GoogleCloudStorageFileSystemTestBase {
+
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.HTTP_API_CLIENT;
+    super.before();
+  }
+}
diff --git a/.../test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/.../test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java
@@ -423,7 +423,8 @@ public void testGetAndListFileInfo() throws Exception {
     // At o1.
     validateListFileInfo(testBucket, "o1", /* expectedToExist= */ true, "o1");
     if (getClass().equals(GoogleCloudStorageFileSystemIntegrationTest.class)
-        || getClass().equals(GoogleCloudStorageFileSystemTest.class)) {
+        || getClass().equals(GoogleCloudStorageFileSystemHTTPClientTest.class)
+        || getClass().equals(GoogleCloudStorageFileSystemJavaStorageClientTest.class)) {
       validateListFileInfo(testBucket, "o1/", /* expectedToExist= */ false);
     } else {
       validateListFileInfo(testBucket, "o1/", /* expectedToExist= */ true, "o1");
@@ -438,7 +439,8 @@ public void testGetAndListFileInfo() throws Exception {
     // At d1/o12.
     validateListFileInfo(testBucket, "d1/o12", /* expectedToExist= */ true, "d1/o12");
     if (getClass().equals(GoogleCloudStorageFileSystemIntegrationTest.class)
-        || getClass().equals(GoogleCloudStorageFileSystemTest.class)) {
+        || getClass().equals(GoogleCloudStorageFileSystemHTTPClientTest.class)
+        || getClass().equals(GoogleCloudStorageFileSystemJavaStorageClientTest.class)) {
       validateListFileInfo(testBucket, "d1/o12/", /* expectedToExist= */ false);
     } else {
       validateListFileInfo(testBucket, "d1/o12/", /* expectedToExist= */ true, "d1/o12");

diff --git a/...java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java b/...java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.gcsio;
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GoogleCloudStorageFileSystemJavaStorageClientTest
+    extends GoogleCloudStorageFileSystemTestBase {
+
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.STORAGE_CLIENT;
+    super.before();
+  }
+
+  @Override
+  @Ignore("DirectPath is not supported with null credentials")
+  @Test
+  public void testConstructor() throws IOException {}
+
+  @Override
+  @Ignore("DirectPath is not supported with null credentials")
+  @Test
+  public void testClientType() {}
+}
diff --git a/...sio/GoogleCloudStorageFileSystemTest.java → ...GoogleCloudStorageFileSystemTestBase.java b/...sio/GoogleCloudStorageFileSystemTest.java → ...GoogleCloudStorageFileSystemTestBase.java
@@ -33,16 +33,14 @@
 import java.util.logging.Logger;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /**
  * The unittest version of {@code GoogleCloudStorageFileSystemIntegrationTest}; the external
  * GoogleCloudStorage dependency is replaced by an in-memory version which mimics the same
  * bucket/object semantics.
  */
-@RunWith(Parameterized.class)
-public class GoogleCloudStorageFileSystemTest extends GoogleCloudStorageFileSystemIntegrationTest {
+public abstract class GoogleCloudStorageFileSystemTestBase
+    extends GoogleCloudStorageFileSystemIntegrationTest {
 
   @Before
   public void before() throws Exception {

diff --git a/pom.xml b/pom.xml
@@ -91,17 +91,17 @@
     <google.api-storage.version>v1-rev20220705-${google.api-client-libraries.version}</google.api-storage.version>
     <google.auth.version>1.12.1</google.auth.version>
     <google.auto-value.version>1.10.1</google.auto-value.version>
-    <google.cloud-bigquerystorage.version>2.28.4</google.cloud-bigquerystorage.version>
+    <google.cloud-bigquerystorage.version>2.36.0</google.cloud-bigquerystorage.version>
     <google.cloud-core.version>2.5.4</google.cloud-core.version>
-    <google.cloud-storage.bom.version>2.17.2</google.cloud-storage.bom.version>
+    <google.cloud-storage.bom.version>2.22.1</google.cloud-storage.bom.version>
     <google.flogger.version>0.7.1</google.flogger.version>
     <google.gax.version>2.19.5</google.gax.version>
     <google.gson.version>2.8.9</google.gson.version>
     <google.guava.version>31.1-jre</google.guava.version>
     <google.http-client.version>1.42.3</google.http-client.version>
     <google.oauth-client.version>1.34.1</google.oauth-client.version>
     <google.protobuf.version>3.21.9</google.protobuf.version>
-    <grpc.version>1.52.1</grpc.version>
+    <grpc.version>1.54.0</grpc.version>
     <hadoop.two.version>2.10.2</hadoop.two.version>
     <hadoop.three.version>3.2.4</hadoop.three.version>
     <opencensus.version>0.31.0</opencensus.version>

diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh
@@ -75,5 +75,7 @@ fi
 export GCS_TEST_PRIVATE_KEYFILE
 export HDFS_ROOT=file:///tmp
 export RUN_INTEGRATION_TESTS=true
+# Env variable to enable direct path over gRPC
+export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true
 
 ./mvnw -B -e -T1C "-P${HADOOP_VERSION}" -Pintegration-test clean verify "${@:5}"