upgrade java-storage (#1001) (#1003)

GoogleCloudDataproc · May 20, 2023 · f8f07f3 · f8f07f3
1 parent 0c5bf33
commit f8f07f3
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 7 deletions.
diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml
@@ -44,6 +44,7 @@ steps:
       - 'VCS_TAG=$TAG_NAME'
       - 'CI_BUILD_ID=$BUILD_ID'
       - 'TRAFFIC_DIRECTOR_ENABLED=false'
+      - 'GCS_TEST_DIRECT_PATH_PREFERRED=false'
 
   # 5. Run Hadoop 3 integration tests concurrently with Hadoop 2 integration tests
   - name: 'gcr.io/$PROJECT_ID/dataproc-hadoop-connectors-presubmit'
@@ -59,9 +60,10 @@ steps:
       - 'VCS_TAG=$TAG_NAME'
       - 'CI_BUILD_ID=$BUILD_ID'
       - 'TRAFFIC_DIRECTOR_ENABLED=false'
+      - 'GCS_TEST_DIRECT_PATH_PREFERRED=false'
 
-# Tests take on average 25 minutes to run
-timeout: 2400s
+# Tests take on average 40 minutes to run
+timeout: 3000s
 
 options:
   machineType: 'N1_HIGHCPU_32'

diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md
@@ -2,6 +2,8 @@
 
 ## Next
 
+1. Upgrade java-storage to 2.22.2
+
 ## 2.2.13 - 2023-05-08
 
 1.  Upgrade Hadoop to 2.10.2/3.2.4.

diff --git a/...src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java b/...src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java
@@ -58,6 +58,8 @@ public static Configuration getTestConfig() {
           testConf.getPrivateKeyFile());
     }
 
+    config.setBoolean("fs.gs.grpc.directpath.enable", testConf.isDirectPathPreferred());
+
     return config;
   }
 

diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java
@@ -21,8 +21,8 @@ public abstract class TestConfiguration {
   public static final String GCS_TEST_PRIVATE_KEYFILE = "GCS_TEST_PRIVATE_KEYFILE";
 
   public static final String GCS_TEST_JSON_KEYFILE = "GCS_TEST_JSON_KEYFILE";
-
   public static final String TRAFFIC_DIRECTOR_ENABLED = "TRAFFIC_DIRECTOR_ENABLED";
+  public static final String GCS_TEST_DIRECT_PATH_PREFERRED = "GCS_TEST_DIRECT_PATH_PREFERRED";
 
   /** Environment-based test configuration. */
   public static class EnvironmentBasedTestConfiguration extends TestConfiguration {
@@ -55,6 +55,16 @@ public boolean isTrafficDirector() {
       }
       return true;
     }
+
+    @Override
+    public boolean isDirectPathPreferred() {
+      String envVar = System.getenv(GCS_TEST_DIRECT_PATH_PREFERRED);
+      // if env variable is not configured default behaviour is to attempt directPath
+      if (envVar == null) {
+        return true;
+      }
+      return Boolean.parseBoolean(envVar);
+    }
   }
 
   public static TestConfiguration getInstance() {
@@ -74,4 +84,6 @@ private static class LazyHolder {
   public abstract String getServiceAccountJsonKeyFile();
 
   public abstract boolean isTrafficDirector();
+
+  public abstract boolean isDirectPathPreferred();
 }
diff --git a/.../test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/.../test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java
@@ -108,6 +108,7 @@ public void before() throws Exception {
               GoogleCloudStorageOptions.builder()
                   .setAppName(GoogleCloudStorageTestHelper.APP_NAME)
                   .setProjectId(projectId)
+                  .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred())
                   .setWriteChannelOptions(
                       AsyncWriteChannelOptions.builder()
                           .setUploadChunkSize(UPLOAD_CHUNK_SIZE_DEFAULT)

diff --git a/...src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java b/...src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java
@@ -115,6 +115,7 @@ public static Credentials getCredentials() throws IOException {
   public static GoogleCloudStorageOptions.Builder getStandardOptionBuilder() {
     return GoogleCloudStorageOptions.builder()
         .setAppName(GoogleCloudStorageTestHelper.APP_NAME)
+        .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred())
         .setProjectId(checkNotNull(TestConfiguration.getInstance().getProjectId()));
   }
 

diff --git a/pom.xml b/pom.xml
@@ -93,9 +93,9 @@
     <google.auto-value.version>1.10.1</google.auto-value.version>
     <google.cloud-bigquerystorage.version>2.36.0</google.cloud-bigquerystorage.version>
     <google.cloud-core.version>2.5.4</google.cloud-core.version>
-    <google.cloud-storage.bom.version>2.22.1</google.cloud-storage.bom.version>
+    <google.cloud-storage.bom.version>2.22.2</google.cloud-storage.bom.version>
     <google.flogger.version>0.7.1</google.flogger.version>
-    <google.gax.version>2.19.5</google.gax.version>
+    <google.gax.version>2.27.0</google.gax.version>
     <google.gson.version>2.8.9</google.gson.version>
     <google.guava.version>31.1-jre</google.guava.version>
     <google.http-client.version>1.42.3</google.http-client.version>

diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh
@@ -75,7 +75,5 @@ fi
 export GCS_TEST_PRIVATE_KEYFILE
 export HDFS_ROOT=file:///tmp
 export RUN_INTEGRATION_TESTS=true
-# Env variable to enable direct path over gRPC
-export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true
 
 ./mvnw -B -e -T1C "-P${HADOOP_VERSION}" -Pintegration-test clean verify "${@:5}"
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,6 +2,8 @@ @@
     ## Next
+. Upgrade java-storage to 2.22.2
     ## 2.2.13 - 2023-05-08
 .  Upgrade Hadoop to 2.10.2/3.2.4.
@@ Expand Down @@