From 0c0d8721072ea9ce650cee25321c48e3163c6385 Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Tue, 16 May 2023 21:44:17 +0530 Subject: [PATCH 1/8] upgrade java-storage --- gcs/CHANGES.md | 2 ++ pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md index b99bbd9823..5c7288941b 100644 --- a/gcs/CHANGES.md +++ b/gcs/CHANGES.md @@ -120,6 +120,8 @@ 1. Upgrade Hadoop to 3.3.5. +1. Upgrade java-storage to 2.22.2 + ### 2.2.2 - 2021-06-25 1. Support footer prefetch in gRPC read channel. diff --git a/pom.xml b/pom.xml index 479b1b7bfc..a39c03e9e1 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 1.14.0 1.10.1 2.9.0 - 2.22.1 + 2.22.2 2.16 0.7.4 2.20.1 From d88ac23fcf8a078cd45cabe5f785deb259f481a6 Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Wed, 17 May 2023 18:25:14 +0530 Subject: [PATCH 2/8] update integration test for directPath --- .../GoogleHadoopFileSystemIntegrationHelper.java | 2 ++ .../hadoop/gcsio/testing/TestConfiguration.java | 14 ++++++++++++++ ...oogleCloudStorageFileSystemIntegrationTest.java | 10 ++++++++++ 3 files changed, 26 insertions(+) diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java index 843b20b5cd..f7074fd713 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java @@ -54,6 +54,8 @@ public static Configuration getTestConfig() { "fs.gs.auth.service.account.json.keyfile", testConf.getServiceAccountJsonKeyFile()); } + config.setBoolean("fs.gs.grpc.directpath.enable", testConf.isDirectPathPreferred()); + return config; } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java index 4638484ada..6b06891744 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java @@ -21,6 +21,8 @@ public abstract class TestConfiguration { public static final String GCS_TEST_PROJECT_ID = "GCS_TEST_PROJECT_ID"; public static final String GCS_TEST_JSON_KEYFILE = "GCS_TEST_JSON_KEYFILE"; + public static final String GCS_TEST_DIRECT_PATH_PREFERRED = "GCS_TEST_DIRECT_PATH_PREFERRED"; + /** Environment-based test configuration. */ public static class EnvironmentBasedTestConfiguration extends TestConfiguration { @Override @@ -32,6 +34,16 @@ public String getProjectId() { public String getServiceAccountJsonKeyFile() { return System.getenv(GCS_TEST_JSON_KEYFILE); } + + @Override + public boolean isDirectPathPreferred() { + String envVar = System.getProperty(GCS_TEST_DIRECT_PATH_PREFERRED); + // if env variable is not configured default behaviour is to attempt directPath + if (envVar == null) { + return true; + } + return Boolean.parseBoolean(envVar); + } } public static TestConfiguration getInstance() { @@ -45,4 +57,6 @@ private static class LazyHolder { public abstract String getProjectId(); public abstract String getServiceAccountJsonKeyFile(); + + public abstract boolean isDirectPathPreferred(); } diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java index d6e4e0422b..ea27796c65 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java @@ -107,6 +107,7 @@ public void before() throws Exception { GoogleCloudStorageOptions.builder() .setAppName(GoogleCloudStorageTestHelper.APP_NAME) .setProjectId(projectId) + .setDirectPathPreferred(isDirectPathAllowed()) .setWriteChannelOptions( AsyncWriteChannelOptions.builder() .setUploadChunkSize(64 * 1024 * 1024) @@ -133,6 +134,15 @@ public void after() throws IOException { } } + public boolean isDirectPathAllowed() { + String envVar = System.getProperty("GCS_TEST_DIRECT_PATH_PREFERRED"); + // if env variable is not configured default behaviour is to attempt directPath + if (envVar == null) { + return true; + } + return Boolean.parseBoolean(envVar); + } + public void postCreateInit() throws IOException { postCreateInit(new GoogleCloudStorageFileSystemIntegrationHelper(gcsfs)); } From f1bf8d62729fb083eea23a980b44759ec058687f Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Wed, 17 May 2023 21:29:20 +0530 Subject: [PATCH 3/8] fixed gcsImplTests --- .../GoogleCloudStorageFileSystemIntegrationTest.java | 11 +---------- .../integration/GoogleCloudStorageTestHelper.java | 1 + 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java index ea27796c65..c8f0c223ac 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java @@ -107,7 +107,7 @@ public void before() throws Exception { GoogleCloudStorageOptions.builder() .setAppName(GoogleCloudStorageTestHelper.APP_NAME) .setProjectId(projectId) - .setDirectPathPreferred(isDirectPathAllowed()) + .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred()) .setWriteChannelOptions( AsyncWriteChannelOptions.builder() .setUploadChunkSize(64 * 1024 * 1024) @@ -134,15 +134,6 @@ public void after() throws IOException { } } - public boolean isDirectPathAllowed() { - String envVar = System.getProperty("GCS_TEST_DIRECT_PATH_PREFERRED"); - // if env variable is not configured default behaviour is to attempt directPath - if (envVar == null) { - return true; - } - return Boolean.parseBoolean(envVar); - } - public void postCreateInit() throws IOException { postCreateInit(new GoogleCloudStorageFileSystemIntegrationHelper(gcsfs)); } diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java index f5ac76cb8b..b9a4a4815e 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java @@ -110,6 +110,7 @@ public static Credentials getCredentials() throws IOException { public static GoogleCloudStorageOptions.Builder getStandardOptionBuilder() { return GoogleCloudStorageOptions.builder() .setAppName(GoogleCloudStorageTestHelper.APP_NAME) + .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred()) .setProjectId(checkNotNull(TestConfiguration.getInstance().getProjectId())); } From 36c8cc13fa76731f2657b37ee7db30336981165b Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Fri, 19 May 2023 11:44:29 +0530 Subject: [PATCH 4/8] upgrade gax-grpc version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a39c03e9e1..fe63a30a33 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ 2.22.2 2.16 0.7.4 - 2.20.1 + 2.27.0 2.10 31.1-jre 1.42.3 From c6de9708b396bda381e3de32720923b1ec2192fe Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Fri, 19 May 2023 13:07:16 +0530 Subject: [PATCH 5/8] fixed env property --- .../google/cloud/hadoop/gcsio/testing/TestConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java index 6b06891744..9dc7ebde38 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java @@ -37,7 +37,7 @@ public String getServiceAccountJsonKeyFile() { @Override public boolean isDirectPathPreferred() { - String envVar = System.getProperty(GCS_TEST_DIRECT_PATH_PREFERRED); + String envVar = System.getenv(GCS_TEST_DIRECT_PATH_PREFERRED); // if env variable is not configured default behaviour is to attempt directPath if (envVar == null) { return true; From b2b7e63c835c4dda1ebd74968b099f29aaa78ba1 Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Fri, 19 May 2023 13:57:09 +0530 Subject: [PATCH 6/8] diable directpath over cloudBuild --- cloudbuild/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml index 453da99de6..c520032fa4 100644 --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -44,6 +44,7 @@ steps: - 'VCS_COMMIT_ID=$COMMIT_SHA' - 'VCS_TAG=$TAG_NAME' - 'CI_BUILD_ID=$BUILD_ID' + - 'GCS_TEST_DIRECT_PATH_PREFERRED=false' # Tests take on average 25 minutes to run timeout: 2400s From 65c3d79aa3514a764c9a66f5cb7d93ffa3418ff1 Mon Sep 17 00:00:00 2001 From: Ravi Dutt Singh Date: Fri, 19 May 2023 15:36:02 +0530 Subject: [PATCH 7/8] remove xds env variable --- tools/run_integration_tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh index c02927a573..13569bd111 100755 --- a/tools/run_integration_tests.sh +++ b/tools/run_integration_tests.sh @@ -71,7 +71,5 @@ fi export GCS_TEST_JSON_KEYFILE export HDFS_ROOT=file:///tmp export RUN_INTEGRATION_TESTS=true -# Env variable to enable direct path over gRPC -export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true ./mvnw -B -e -T1C -Pintegration-test clean verify "${@:3}" From d840afb7e18c938a1fdfba40ac8fbfd56e81eaac Mon Sep 17 00:00:00 2001 From: Igor Dvorzhak Date: Fri, 19 May 2023 16:20:11 -0700 Subject: [PATCH 8/8] Update CHANGES.md --- gcs/CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md index 5c7288941b..86f3223cb9 100644 --- a/gcs/CHANGES.md +++ b/gcs/CHANGES.md @@ -120,7 +120,7 @@ 1. Upgrade Hadoop to 3.3.5. -1. Upgrade java-storage to 2.22.2 +1. Upgrade java-storage to 2.22.2 ### 2.2.2 - 2021-06-25