diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml index c313d676c9..7cf51313c1 100644 --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -44,6 +44,7 @@ steps: - 'VCS_TAG=$TAG_NAME' - 'CI_BUILD_ID=$BUILD_ID' - 'TRAFFIC_DIRECTOR_ENABLED=false' + - 'GCS_TEST_DIRECT_PATH_PREFERRED=false' # 5. Run Hadoop 3 integration tests concurrently with Hadoop 2 integration tests - name: 'gcr.io/$PROJECT_ID/dataproc-hadoop-connectors-presubmit' @@ -59,9 +60,10 @@ steps: - 'VCS_TAG=$TAG_NAME' - 'CI_BUILD_ID=$BUILD_ID' - 'TRAFFIC_DIRECTOR_ENABLED=false' + - 'GCS_TEST_DIRECT_PATH_PREFERRED=false' -# Tests take on average 25 minutes to run -timeout: 2400s +# Tests take on average 40 minutes to run +timeout: 3000s options: machineType: 'N1_HIGHCPU_32' diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md index e109dbb66f..3ddd2fe6a7 100644 --- a/gcs/CHANGES.md +++ b/gcs/CHANGES.md @@ -2,6 +2,8 @@ ## Next +1. Upgrade java-storage to 2.22.2 + ## 2.2.13 - 2023-05-08 1. Upgrade Hadoop to 2.10.2/3.2.4. diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java index e333639945..49e8a5c636 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java @@ -58,6 +58,8 @@ public static Configuration getTestConfig() { testConf.getPrivateKeyFile()); } + config.setBoolean("fs.gs.grpc.directpath.enable", testConf.isDirectPathPreferred()); + return config; } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java index 5d800dc002..9aa2d7a759 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java @@ -21,8 +21,8 @@ public abstract class TestConfiguration { public static final String GCS_TEST_PRIVATE_KEYFILE = "GCS_TEST_PRIVATE_KEYFILE"; public static final String GCS_TEST_JSON_KEYFILE = "GCS_TEST_JSON_KEYFILE"; - public static final String TRAFFIC_DIRECTOR_ENABLED = "TRAFFIC_DIRECTOR_ENABLED"; + public static final String GCS_TEST_DIRECT_PATH_PREFERRED = "GCS_TEST_DIRECT_PATH_PREFERRED"; /** Environment-based test configuration. */ public static class EnvironmentBasedTestConfiguration extends TestConfiguration { @@ -55,6 +55,16 @@ public boolean isTrafficDirector() { } return true; } + + @Override + public boolean isDirectPathPreferred() { + String envVar = System.getenv(GCS_TEST_DIRECT_PATH_PREFERRED); + // if env variable is not configured default behaviour is to attempt directPath + if (envVar == null) { + return true; + } + return Boolean.parseBoolean(envVar); + } } public static TestConfiguration getInstance() { @@ -74,4 +84,6 @@ private static class LazyHolder { public abstract String getServiceAccountJsonKeyFile(); public abstract boolean isTrafficDirector(); + + public abstract boolean isDirectPathPreferred(); } diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java index 3e4d62e1ae..4fb0f5b158 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java @@ -108,6 +108,7 @@ public void before() throws Exception { GoogleCloudStorageOptions.builder() .setAppName(GoogleCloudStorageTestHelper.APP_NAME) .setProjectId(projectId) + .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred()) .setWriteChannelOptions( AsyncWriteChannelOptions.builder() .setUploadChunkSize(UPLOAD_CHUNK_SIZE_DEFAULT) diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java index 1742ffc003..a80e382094 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java @@ -115,6 +115,7 @@ public static Credentials getCredentials() throws IOException { public static GoogleCloudStorageOptions.Builder getStandardOptionBuilder() { return GoogleCloudStorageOptions.builder() .setAppName(GoogleCloudStorageTestHelper.APP_NAME) + .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred()) .setProjectId(checkNotNull(TestConfiguration.getInstance().getProjectId())); } diff --git a/pom.xml b/pom.xml index 4481563a87..9147f7f9b7 100644 --- a/pom.xml +++ b/pom.xml @@ -93,9 +93,9 @@ 1.10.1 2.36.0 2.5.4 - 2.22.1 + 2.22.2 0.7.1 - 2.19.5 + 2.27.0 2.8.9 31.1-jre 1.42.3 diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh index ad14c80ae9..c2d23a1d9a 100755 --- a/tools/run_integration_tests.sh +++ b/tools/run_integration_tests.sh @@ -75,7 +75,5 @@ fi export GCS_TEST_PRIVATE_KEYFILE export HDFS_ROOT=file:///tmp export RUN_INTEGRATION_TESTS=true -# Env variable to enable direct path over gRPC -export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true ./mvnw -B -e -T1C "-P${HADOOP_VERSION}" -Pintegration-test clean verify "${@:5}"