From 5ceaffb3783c83892f47634fd67c2526b4eb2e3f Mon Sep 17 00:00:00 2001 From: Marc Cenac Date: Tue, 24 Sep 2024 10:35:15 -0500 Subject: [PATCH] Add support for WASBS to native Azure file system implementation --- .../io/trino/filesystem/azure/AzureLocation.java | 12 ++++++------ .../io/trino/filesystem/azure/TestAzureLocation.java | 7 +++++-- .../trino/filesystem/manager/FileSystemModule.java | 1 + 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureLocation.java b/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureLocation.java index f3222942426e..aec71e829af0 100644 --- a/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureLocation.java +++ b/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureLocation.java @@ -23,8 +23,8 @@ class AzureLocation { - private static final String INVALID_ABFS_LOCATION_MESSAGE = "Invalid Azure ABFS location. Expected form is 'abfs://[@].dfs./': %s"; - private static final String INVALID_WASB_LOCATION_MESSAGE = "Invalid Azure WASB location. Expected form is 'wasb://[@].blob./': %s"; + private static final String INVALID_ABFS_LOCATION_MESSAGE = "Invalid Azure ABFS location. Expected form is 'abfs[s]://[@].dfs./': %s"; + private static final String INVALID_WASB_LOCATION_MESSAGE = "Invalid Azure WASB location. Expected form is 'wasb[s]://[@].blob./': %s"; // https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules private static final CharMatcher CONTAINER_VALID_CHARACTERS = CharMatcher.inRange('a', 'z').or(CharMatcher.inRange('0', '9')).or(CharMatcher.is('-')); @@ -40,18 +40,18 @@ class AzureLocation *

* Locations use the * ABFS URI syntax: - * 

{@code abfs://@.dfs./}
+ * 
{@code abfs[s]://@.dfs./}
*/ public AzureLocation(Location location) { this.location = requireNonNull(location, "location is null"); - // abfss and wasb are also supported but not documented + // wasb and wasbs are also supported but not documented scheme = location.scheme().orElseThrow(() -> new IllegalArgumentException(String.format(INVALID_ABFS_LOCATION_MESSAGE, location))); String invalidLocationMessage; if ("abfs".equals(scheme) || "abfss".equals(scheme)) { invalidLocationMessage = INVALID_ABFS_LOCATION_MESSAGE; } - else if ("wasb".equals(scheme)) { + else if ("wasb".equals(scheme) || "wasbs".equals(scheme)) { invalidLocationMessage = INVALID_WASB_LOCATION_MESSAGE; } else { @@ -86,7 +86,7 @@ else if ("wasb".equals(scheme)) { this.location); this.account = host.substring(0, accountSplit); - // abfs[s] host must contain ".dfs.", and wasb host must contain ".blob." before endpoint + // abfs[s] host must contain ".dfs.", and wasb[s] host must contain ".blob." before endpoint if (scheme.equals("abfs") || scheme.equals("abfss")) { checkArgument(host.substring(accountSplit).startsWith(".dfs."), invalidLocationMessage, location); // endpoint does not include dfs diff --git a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureLocation.java b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureLocation.java index 11dab8dd414d..7f3feee4ec54 100644 --- a/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureLocation.java +++ b/lib/trino-filesystem-azure/src/test/java/io/trino/filesystem/azure/TestAzureLocation.java @@ -29,6 +29,7 @@ void test() assertValid("abfs://container@account.dfs.core.windows.net/some/path/file", "account", "container", "some/path/file", "abfs", "core.windows.net"); assertValid("abfss://container@account.dfs.core.windows.net/some/path/file", "account", "container", "some/path/file", "abfss", "core.windows.net"); assertValid("wasb://container@account.blob.core.windows.net/some/path/file", "account", "container", "some/path/file", "wasb", "core.windows.net"); + assertValid("wasbs://container@account.blob.core.windows.net/some/path/file", "account", "container", "some/path/file", "wasbs", "core.windows.net"); assertValid("abfs://container-stuff@account.dfs.core.windows.net/some/path/file", "account", "container-stuff", "some/path/file", "abfs", "core.windows.net"); assertValid("abfs://container2@account.dfs.core.windows.net/some/path/file", "account", "container2", "some/path/file", "abfs", "core.windows.net"); @@ -41,16 +42,18 @@ void test() assertValid("abfs://container@account.dfs.core.usgovcloudapi.net/some/path/file", "account", "container", "some/path/file", "abfs", "core.usgovcloudapi.net"); assertValid("abfss://container@account.dfs.core.usgovcloudapi.net/some/path/file", "account", "container", "some/path/file", "abfss", "core.usgovcloudapi.net"); assertValid("wasb://container@account.blob.core.usgovcloudapi.net/some/path/file", "account", "container", "some/path/file", "wasb", "core.usgovcloudapi.net"); + assertValid("wasbs://container@account.blob.core.usgovcloudapi.net/some/path/file", "account", "container", "some/path/file", "wasbs", "core.usgovcloudapi.net"); - // abfs[s] host must contain ".dfs.", and wasb host must contain ".blob." before endpoint + // abfs[s] host must contain ".dfs.", and wasb[s] host must contain ".blob." before endpoint assertInvalid("abfs://container@account.invalid.core.usgovcloudapi.net/some/path/file"); assertInvalid("abfss://container@account.invalid.core.usgovcloudapi.net/some/path/file"); assertInvalid("wasb://container@account.invalid.core.usgovcloudapi.net/some/path/file"); assertInvalid("abfs://container@account.blob.core.usgovcloudapi.net/some/path/file"); assertInvalid("abfss://container@account.blob.core.usgovcloudapi.net/some/path/file"); assertInvalid("wasb://container@account.dfs.core.usgovcloudapi.net/some/path/file"); + assertInvalid("wasbs://container@account.dfs.core.usgovcloudapi.net/some/path/file"); - // only abfs, abfss, and wasb schemes allowed + // only abfs, abfss, wasb, wasbs schemes allowed assertInvalid("https://container@account.dfs.core.windows.net/some/path/file"); // host must have at least to labels diff --git a/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java b/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java index d0e980ec5409..c28f1fe7dbda 100644 --- a/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java +++ b/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java @@ -101,6 +101,7 @@ protected void setup(Binder binder) factories.addBinding("abfs").to(AzureFileSystemFactory.class); factories.addBinding("abfss").to(AzureFileSystemFactory.class); factories.addBinding("wasb").to(AzureFileSystemFactory.class); + factories.addBinding("wasbs").to(AzureFileSystemFactory.class); } if (config.isNativeS3Enabled()) {