diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml
index c313d676c9..7cf51313c1 100644
--- a/cloudbuild/cloudbuild.yaml
+++ b/cloudbuild/cloudbuild.yaml
@@ -44,6 +44,7 @@ steps:
- 'VCS_TAG=$TAG_NAME'
- 'CI_BUILD_ID=$BUILD_ID'
- 'TRAFFIC_DIRECTOR_ENABLED=false'
+ - 'GCS_TEST_DIRECT_PATH_PREFERRED=false'
# 5. Run Hadoop 3 integration tests concurrently with Hadoop 2 integration tests
- name: 'gcr.io/$PROJECT_ID/dataproc-hadoop-connectors-presubmit'
@@ -59,9 +60,10 @@ steps:
- 'VCS_TAG=$TAG_NAME'
- 'CI_BUILD_ID=$BUILD_ID'
- 'TRAFFIC_DIRECTOR_ENABLED=false'
+ - 'GCS_TEST_DIRECT_PATH_PREFERRED=false'
-# Tests take on average 25 minutes to run
-timeout: 2400s
+# Tests take on average 40 minutes to run
+timeout: 3000s
options:
machineType: 'N1_HIGHCPU_32'
diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md
index e109dbb66f..3ddd2fe6a7 100644
--- a/gcs/CHANGES.md
+++ b/gcs/CHANGES.md
@@ -2,6 +2,8 @@
## Next
+1. Upgrade java-storage to 2.22.2
+
## 2.2.13 - 2023-05-08
1. Upgrade Hadoop to 2.10.2/3.2.4.
diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java
index e333639945..49e8a5c636 100644
--- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java
+++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java
@@ -58,6 +58,8 @@ public static Configuration getTestConfig() {
testConf.getPrivateKeyFile());
}
+ config.setBoolean("fs.gs.grpc.directpath.enable", testConf.isDirectPathPreferred());
+
return config;
}
diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java
index 5d800dc002..9aa2d7a759 100644
--- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java
+++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java
@@ -21,8 +21,8 @@ public abstract class TestConfiguration {
public static final String GCS_TEST_PRIVATE_KEYFILE = "GCS_TEST_PRIVATE_KEYFILE";
public static final String GCS_TEST_JSON_KEYFILE = "GCS_TEST_JSON_KEYFILE";
-
public static final String TRAFFIC_DIRECTOR_ENABLED = "TRAFFIC_DIRECTOR_ENABLED";
+ public static final String GCS_TEST_DIRECT_PATH_PREFERRED = "GCS_TEST_DIRECT_PATH_PREFERRED";
/** Environment-based test configuration. */
public static class EnvironmentBasedTestConfiguration extends TestConfiguration {
@@ -55,6 +55,16 @@ public boolean isTrafficDirector() {
}
return true;
}
+
+ @Override
+ public boolean isDirectPathPreferred() {
+ String envVar = System.getenv(GCS_TEST_DIRECT_PATH_PREFERRED);
+ // if env variable is not configured default behaviour is to attempt directPath
+ if (envVar == null) {
+ return true;
+ }
+ return Boolean.parseBoolean(envVar);
+ }
}
public static TestConfiguration getInstance() {
@@ -74,4 +84,6 @@ private static class LazyHolder {
public abstract String getServiceAccountJsonKeyFile();
public abstract boolean isTrafficDirector();
+
+ public abstract boolean isDirectPathPreferred();
}
diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java
index 3e4d62e1ae..4fb0f5b158 100644
--- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java
+++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java
@@ -108,6 +108,7 @@ public void before() throws Exception {
GoogleCloudStorageOptions.builder()
.setAppName(GoogleCloudStorageTestHelper.APP_NAME)
.setProjectId(projectId)
+ .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred())
.setWriteChannelOptions(
AsyncWriteChannelOptions.builder()
.setUploadChunkSize(UPLOAD_CHUNK_SIZE_DEFAULT)
diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java
index 1742ffc003..a80e382094 100644
--- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java
+++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java
@@ -115,6 +115,7 @@ public static Credentials getCredentials() throws IOException {
public static GoogleCloudStorageOptions.Builder getStandardOptionBuilder() {
return GoogleCloudStorageOptions.builder()
.setAppName(GoogleCloudStorageTestHelper.APP_NAME)
+ .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred())
.setProjectId(checkNotNull(TestConfiguration.getInstance().getProjectId()));
}
diff --git a/pom.xml b/pom.xml
index 4481563a87..9147f7f9b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,9 +93,9 @@
1.10.1
2.36.0
2.5.4
- 2.22.1
+ 2.22.2
0.7.1
- 2.19.5
+ 2.27.0
2.8.9
31.1-jre
1.42.3
diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh
index ad14c80ae9..c2d23a1d9a 100755
--- a/tools/run_integration_tests.sh
+++ b/tools/run_integration_tests.sh
@@ -75,7 +75,5 @@ fi
export GCS_TEST_PRIVATE_KEYFILE
export HDFS_ROOT=file:///tmp
export RUN_INTEGRATION_TESTS=true
-# Env variable to enable direct path over gRPC
-export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true
./mvnw -B -e -T1C "-P${HADOOP_VERSION}" -Pintegration-test clean verify "${@:5}"