From f4ab98fde61433aa0b82ae2f3d754c25631b5d4f Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Mon, 19 Jul 2021 20:41:54 +0200 Subject: [PATCH 01/19] fix custom host logic check LDEV-3594 --- .../org/lucee/extension/resource/s3/S3ResourceProvider.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index c19caa5..56dd21c 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -225,10 +225,10 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, } else { String _host = path.substring(0, index); - if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) { + //if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) { properties.setHost(_host); path = path.substring(index); - } + //} } // get from system.properties/env.var @@ -349,4 +349,4 @@ public Map getArguments() { public char getSeparator() { return '/'; } -} \ No newline at end of file +} From f6ff40440ef48a780ef9c665fa602648e8a8a894 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Tue, 20 Jul 2021 18:17:54 +0200 Subject: [PATCH 02/19] improve s3 resource url parsing for custom host urls this includes initial support for parsing s3://miniadmin:minioadmin@http://localhost:9000/bucket which requires setting for jetS3t - s3service.https-only = false "s3service.s3-endpoint-http-port = 9000 Lucee 6 has support for setting env vars for testing this https://github.com/lucee/Lucee/blob/6.0/test/_setupTestServices.cfc#L118 https://github.com/lucee/Lucee/commit/aae583ac400f9ac9a1ef7791a58099606b0ce756 --- .../src/org/lucee/extension/resource/s3/S3.java | 17 ++++++++++++++++- .../resource/s3/S3ResourceProvider.java | 14 ++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index f12438b..072f5e9 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -1250,7 +1250,22 @@ private S3Service getS3Service() { if (service == null) { if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) { final Jets3tProperties props = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME); - props.setProperty("s3service.s3-endpoint", host); + int index = host.indexOf(':'); + if (index == -1) props.setProperty("s3service.s3-endpoint", host); // no port, no protocol + else { + int index2 = host.toLowerCase().indexOf("http", 0); // http(s)://localhost:9000 prefix? + if (index2 == -1){ // no http prefix + props.setProperty("s3service.s3-endpoint", host.substring(0, index)); + props.setProperty("s3service.s3-endpoint-http-port", host.substring(index + 1)); + } else { // with http(s) prefix + String protocol = host.substring(0,host.indexOf(":")); + props.setProperty("s3service.https-only", protocol.equalsIgnoreCase("http") ? "false" : "true" ); + String _host = host.substring(host.lastIndexOf("/") + 1); + index = _host.indexOf(':'); + props.setProperty("s3service.s3-endpoint", _host.substring(0, index)); + props.setProperty("s3service.s3-endpoint-http-port", _host.substring(index + 1)); + } + } service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props); } else { diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index 56dd21c..01b4845 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -224,10 +224,20 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, } } else { - String _host = path.substring(0, index); //if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) { + int index2 = path.indexOf('/', index+1); + if (index2 == -1) { + //String _host = path.substring(0, index); + //properties.setHost(_host); + //path = path.substring(index); + } else { + // two slashes in the host string means there's a http prefix on the host (single / due to prettifyPath) + String _host = path.substring(0, index2); // i.e. http:/localhost:9000 properties.setHost(_host); - path = path.substring(index); + host = _host; + path = path.substring(index2); + } + //} } From a5e88af48a74351e0aef5d539dd662669d945816 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 21 Jul 2021 11:04:14 +0200 Subject: [PATCH 03/19] add s3_google profile to GHA --- .github/workflows/main.yml | 2 ++ .../org/lucee/extension/resource/s3/S3ResourceProvider.java | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 46e1fbc..f61366b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,3 +65,5 @@ jobs: S3_CUSTOM_ACCESS_KEY_ID: "minioadmin" S3_CUSTOM_SECRET_KEY: "minioadmin" S3_CUSTOM_HOST: "localhost:9000" + S3_GOOGLE_ACCESS_KEY_ID: ${{ secrets.S3_GOOGLE_SECRET_KEY }} + S3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }} diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index 01b4845..b69c841 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -232,9 +232,8 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, //path = path.substring(index); } else { // two slashes in the host string means there's a http prefix on the host (single / due to prettifyPath) - String _host = path.substring(0, index2); // i.e. http:/localhost:9000 - properties.setHost(_host); - host = _host; + host = path.substring(0, index2); // i.e. http:/localhost:9000 + properties.setHost(host); path = path.substring(index2); } From cabf7da0c82d5cee45fe2f0467a7675f8fd126bb Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 21 Jul 2021 11:07:59 +0200 Subject: [PATCH 04/19] add S3_GOOGLE_HOST to GHA --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f61366b..69b8511 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,3 +67,4 @@ jobs: S3_CUSTOM_HOST: "localhost:9000" S3_GOOGLE_ACCESS_KEY_ID: ${{ secrets.S3_GOOGLE_SECRET_KEY }} S3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }} + S3_GOOGLE_HOST: "storage.googleapis.com" From 5153f05601de9440468e52f88d18f0c85cb764dc Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 21 Jul 2021 11:11:14 +0200 Subject: [PATCH 05/19] limit s3 GHA to run synchronously --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69b8511..dba9db9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,9 @@ name: Java CI -on: [push, pull_request,workflow_dispatch] +on: [push, pull_request, workflow_dispatch] + +concurrency: s3_extension jobs: build: From d4da0d1fb1f76880fe226973c2a1794af9944cab Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 21 Jul 2021 17:40:35 +0200 Subject: [PATCH 06/19] fix custom s3 host url parsing --- .github/workflows/main.yml | 6 +-- .../org/lucee/extension/resource/s3/S3.java | 1 + .../resource/s3/S3ResourceProvider.java | 48 +++++++++---------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dba9db9..362f951 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,10 +63,10 @@ jobs: testLabels: s3 testAdditional: ${{ github.workspace }}/tests S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_ID_TEST }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }} + SS3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }} S3_CUSTOM_ACCESS_KEY_ID: "minioadmin" S3_CUSTOM_SECRET_KEY: "minioadmin" - S3_CUSTOM_HOST: "localhost:9000" + S3_CUSTOM_HOST: "http://localhost:9000" S3_GOOGLE_ACCESS_KEY_ID: ${{ secrets.S3_GOOGLE_SECRET_KEY }} - S3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }} + SS3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }} S3_GOOGLE_HOST: "storage.googleapis.com" diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index a8e82eb..5208303 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -1293,6 +1293,7 @@ private S3Service getS3Service() { index = _host.indexOf(':'); props.setProperty("s3service.s3-endpoint", _host.substring(0, index)); props.setProperty("s3service.s3-endpoint-http-port", _host.substring(index + 1)); + props.setProperty("s3service.disable-dns-buckets", "true"); } } service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props); diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index b69c841..a0b76b6 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -166,9 +166,9 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, storage.setValue(defaultLocation); int atIndex = path.indexOf('@'); - int slashIndex = path.indexOf('/'); + int slashIndex = path.indexOf('/', atIndex); // secret keys can contain / if (slashIndex == -1) { - slashIndex = path.length(); + //slashIndex = path.length(); // never used! path += "/"; } int index; @@ -214,32 +214,30 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, // } } - path = prettifyPath(path.substring(atIndex + 1)); - index = path.indexOf('/'); - properties.setHost(host); - if (index == -1) { - if (path.equalsIgnoreCase(S3.DEFAULT_HOST) || path.equalsIgnoreCase(host)) { - properties.setHost(path); - path = "/"; - } - } - else { - //if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) { - int index2 = path.indexOf('/', index+1); - if (index2 == -1) { - //String _host = path.substring(0, index); - //properties.setHost(_host); - //path = path.substring(index); + String srcPath = path; + if ( (slashIndex - atIndex) == 1 ){ + // key/secret@/bucket + path = prettifyPath(path.substring(atIndex + 1)); + } else { + int doubleIndex = path.indexOf("://", atIndex); // do we have a http(s):// + if (doubleIndex > 0) { + // key/secret@http://localhost:9000/bucket + index = path.indexOf('/', doubleIndex+3 ); + host = path.substring(atIndex+1, index); + path = prettifyPath(path.substring(index)); } else { - // two slashes in the host string means there's a http prefix on the host (single / due to prettifyPath) - host = path.substring(0, index2); // i.e. http:/localhost:9000 - properties.setHost(host); - path = path.substring(index2); + path = prettifyPath(path.substring(atIndex + 1)); + index = path.indexOf('/'); + if (index == -1) { + path = "/"; + } else if ( (slashIndex - atIndex) != 1 ) { + // key/secret@storage.googleapis.com/bucket + host = path.substring(0, index); + path = path.substring(index); + } } - - //} } - + properties.setHost(host); // get from system.properties/env.var if (Util.isEmpty(accessKeyId, true)) { if (Util.isEmpty(mappingName)) { From ff9737d6c5b3410efa656e79ae8624743652cc25 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 21 Jul 2021 21:38:20 +0200 Subject: [PATCH 07/19] fix NPE --- .../lucee/extension/resource/s3/util/ApplicationSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java index b221354..0e6f892 100644 --- a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java +++ b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java @@ -89,6 +89,7 @@ public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) } } final Struct sct = (Struct) bif.invoke(pc, new Object[] { Boolean.TRUE }); + coll = new S3PropertiesCollection(); Struct data; { // read default s3 properties @@ -102,7 +103,6 @@ public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) S3Properties s3prop = toS3Properties(eng, data, null); if (s3prop != null) { - coll = new S3PropertiesCollection(); coll.setDefault(s3prop); propsColl.put(key, coll); } From 7532022baad6f63ca12af28bad04d8ca4cb801f0 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Thu, 22 Jul 2021 13:48:30 +0200 Subject: [PATCH 08/19] avoid NPE --- .../extension/resource/s3/util/ApplicationSettings.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java index 2bbe810..e705128 100644 --- a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java +++ b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java @@ -65,7 +65,10 @@ public ApplicationSettings() { public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) throws PageException { Properties prop = pc.getApplicationContext().getS3(); - String key = "" + prop.hashCode(); + String key; + + if (prop == null) key = "none"; + else key = "" + prop.hashCode(); S3PropertiesCollection existing = propsColl.get(key); if (existing != null) { @@ -90,7 +93,6 @@ public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) } } final Struct sct = (Struct) bif.invoke(pc, new Object[] { Boolean.TRUE }); - coll = new S3PropertiesCollection(); Struct data; { // read default s3 properties @@ -104,6 +106,7 @@ public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) S3Properties s3prop = toS3Properties(eng, data, null); if (s3prop != null) { + coll = new S3PropertiesCollection(); coll.setDefault(s3prop); propsColl.put(key, coll); } From 911af22b5fac4b6abe172be68c64b49f509e7f24 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Thu, 22 Jul 2021 17:14:11 +0200 Subject: [PATCH 09/19] support s3 commonPrefixes LDEV-3631 --- .../java/src/org/lucee/extension/resource/s3/S3.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 5208303..2ed8616 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -561,9 +561,15 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception StorageObject[] objects = chunk == null ? null : chunk.getObjects(); if (objects == null || objects.length == 0) { - exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it - // does - return null; + List commonPrefixes = new ArrayList(); + commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); + if (commonPrefixes.contains(nameDir)){ + // pseudo directory + exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); + } else { + exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it + return null; + } } String targetName; From c11cb5b2181be740bfcb85c4fa5611a01e81bfd4 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Tue, 27 Jul 2021 17:24:56 +0200 Subject: [PATCH 10/19] fix parsing s3://buckname urls without host --- .../org/lucee/extension/resource/s3/S3ResourceProvider.java | 5 ++++- .../extension/resource/s3/util/ApplicationSettings.java | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index b4dae3f..13a9a3f 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -216,7 +216,10 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, } } String srcPath = path; - if ( (slashIndex - atIndex) == 1 ){ + if (atIndex == -1){ + // no host information, skip parsing, ie. s3://bucketname/ etc + path = prettifyPath(path); + } else if ( (slashIndex - atIndex) == 1 ){ // key/secret@/bucket path = prettifyPath(path.substring(atIndex + 1)); } else { diff --git a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java index e705128..d59e444 100644 --- a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java +++ b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java @@ -65,11 +65,8 @@ public ApplicationSettings() { public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) throws PageException { Properties prop = pc.getApplicationContext().getS3(); - String key; + String key = prop == null ? pc.getId() + ":" + pc.getStartTime() : "" + prop.hashCode(); - if (prop == null) key = "none"; - else key = "" + prop.hashCode(); - S3PropertiesCollection existing = propsColl.get(key); if (existing != null) { return existing; From 540c8df152a2d39c83f1ca11da3e896d998cc77f Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Thu, 29 Jul 2021 10:13:26 +0200 Subject: [PATCH 11/19] improve host parsing --- .../org/lucee/extension/resource/s3/S3.java | 40 ++++++++++--------- .../resource/s3/S3ResourceProvider.java | 2 + 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 2ed8616..71ae153 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -42,6 +42,7 @@ import org.lucee.extension.resource.s3.info.S3Info; import org.lucee.extension.resource.s3.info.StorageObjectWrapper; import org.lucee.extension.resource.s3.util.XMLUtil; +//import org.lucee.extension.resource.s3.util.aprint; import lucee.loader.engine.CFMLEngineFactory; import lucee.loader.util.Util; @@ -562,7 +563,11 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception if (objects == null || objects.length == 0) { List commonPrefixes = new ArrayList(); - commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); + if (chunk.getCommonPrefixes().length > 0){ + //String[] cp = chunk.getCommonPrefixes(); + //aprint.o(cp.toString()); + commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); + } if (commonPrefixes.contains(nameDir)){ // pseudo directory exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); @@ -1283,30 +1288,29 @@ private S3Service getS3Service() { if (service == null) { synchronized (getToken(accessKeyId + ":" + secretAccessKey)) { if (service == null) { - if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) { - final Jets3tProperties props = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME); - int index = host.indexOf(':'); - if (index == -1) props.setProperty("s3service.s3-endpoint", host); // no port, no protocol - else { - int index2 = host.toLowerCase().indexOf("http", 0); // http(s)://localhost:9000 prefix? - if (index2 == -1){ // no http prefix - props.setProperty("s3service.s3-endpoint", host.substring(0, index)); - props.setProperty("s3service.s3-endpoint-http-port", host.substring(index + 1)); + final Jets3tProperties props = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME); + props.clearAllProperties(); + //if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) { + int hasPort = host.indexOf(':'); + if (hasPort == -1) { + props.setProperty("s3service.s3-endpoint", host); // no port, no protocol + } else { + int hasHttp = host.toLowerCase().indexOf("http", 0); // http(s)://localhost:9000 prefix? + if (hasHttp == -1){ // no http prefix + props.setProperty("s3service.s3-endpoint", host.substring(0, hasPort)); + props.setProperty("s3service.s3-endpoint-http-port", host.substring(hasPort + 1)); } else { // with http(s) prefix String protocol = host.substring(0,host.indexOf(":")); props.setProperty("s3service.https-only", protocol.equalsIgnoreCase("http") ? "false" : "true" ); String _host = host.substring(host.lastIndexOf("/") + 1); - index = _host.indexOf(':'); - props.setProperty("s3service.s3-endpoint", _host.substring(0, index)); - props.setProperty("s3service.s3-endpoint-http-port", _host.substring(index + 1)); + hasPort = _host.indexOf(':'); + props.setProperty("s3service.s3-endpoint", _host.substring(0, hasPort)); + props.setProperty("s3service.s3-endpoint-http-port", _host.substring(hasPort + 1)); props.setProperty("s3service.disable-dns-buckets", "true"); } } - service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props); - } - else { - service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey)); - } + //} + service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props); } } } diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index 13a9a3f..fdf076c 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -241,6 +241,8 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, } } } + if (host.endsWith("/")) + host = host.substring(0, host.length() - 1); // strip trailing / after hostname with http://localhost:9000/ properties.setHost(host); // get from system.properties/env.var if (Util.isEmpty(accessKeyId, true)) { From 7b1c93d1986bf765307d03212bdc5d86e1e3c3a3 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Thu, 29 Jul 2021 14:00:51 +0200 Subject: [PATCH 12/19] always check common prefixes --- .../org/lucee/extension/resource/s3/S3.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 71ae153..ef78fa7 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -561,22 +561,21 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception long validUntil = System.currentTimeMillis() + timeout; StorageObject[] objects = chunk == null ? null : chunk.getObjects(); - if (objects == null || objects.length == 0) { - List commonPrefixes = new ArrayList(); - if (chunk.getCommonPrefixes().length > 0){ - //String[] cp = chunk.getCommonPrefixes(); - //aprint.o(cp.toString()); - commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); - } + List commonPrefixes = new ArrayList(); + if (chunk.getCommonPrefixes().length > 0){ + //String[] cp = chunk.getCommonPrefixes(); + //aprint.o(cp.toString()); + commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); if (commonPrefixes.contains(nameDir)){ // pseudo directory exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); - } else { - exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it - return null; } } + if (info == null && (objects == null || objects.length == 0)) { + exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it + } + String targetName; StorageObject stoObj = null; // direct match From aaafad27311de7fa9598ac9362b819deb608d819 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Mon, 2 Aug 2021 15:56:54 +0200 Subject: [PATCH 13/19] ensure a s3 host is always prefixed with an @ --- .../src/org/lucee/extension/resource/s3/S3Resource.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java index 9b8ace2..fe2bd61 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java @@ -182,7 +182,11 @@ else if (!Util.isEmpty(s3.getMappingName(), true)) { sb.append(s3.getMappingName()).append("@"); } - if (doHost) sb.append(s3.getHost()); + if (doHost){ + if (sb.substring(sb.length()-1) != "@") + sb.append("@"); + sb.append(s3.getHost()); + } return sb.toString(); } From 4273c70162fddc5b77b74d23277a246abe85697f Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Tue, 3 Aug 2021 15:48:43 +0200 Subject: [PATCH 14/19] don't add host to url when not using custom creds --- source/java/src/org/lucee/extension/resource/s3/S3Resource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java index fe2bd61..187d838 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java @@ -182,7 +182,7 @@ else if (!Util.isEmpty(s3.getMappingName(), true)) { sb.append(s3.getMappingName()).append("@"); } - if (doHost){ + if (doHost && s3.getCustomCredentials()){ if (sb.substring(sb.length()-1) != "@") sb.append("@"); sb.append(s3.getHost()); From c3eedb433fcaf2fccb4667ed8aba43e855867217 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Tue, 3 Aug 2021 15:54:32 +0200 Subject: [PATCH 15/19] try using listObjectsChunked instead of listObjects --- .../org/lucee/extension/resource/s3/S3.java | 146 +++++++++++++----- 1 file changed, 111 insertions(+), 35 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index ef78fa7..92714db 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -42,7 +42,6 @@ import org.lucee.extension.resource.s3.info.S3Info; import org.lucee.extension.resource.s3.info.StorageObjectWrapper; import org.lucee.extension.resource.s3.util.XMLUtil; -//import org.lucee.extension.resource.s3.util.aprint; import lucee.loader.engine.CFMLEngineFactory; import lucee.loader.util.Util; @@ -315,8 +314,10 @@ public List list(boolean recursive, boolean listPseudoFolder) throws S3E * list all allements in a specific bucket * * @param bucketName name of the bucket + * @param objectName name of the object * @param recursive show all objects (recursive==true) or direct kids * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders + * @param onlyChildren * @return * @throws S3Exception * @@ -424,8 +425,8 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea // not cached ValidUntilMap _list = timeout <= 0 || noCache ? null : objects.get(key); if (_list == null || _list.validUntil < System.currentTimeMillis()) { + /* S3Object[] kids = hasObjName ? getS3Service().listObjects(bucketName, nameFile, ",") : getS3Service().listObjects(bucketName); - long validUntil = System.currentTimeMillis() + timeout; _list = new ValidUntilMap(validUntil); objects.put(key, _list); @@ -451,13 +452,64 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil)); } } + */ + + /* ------------------------ chunked, which includes prefixes ------------------------------ */ + StorageObjectsChunk chunk = hasObjName ? + listObjectsChunkedAll(bucketName, nameFile, ",", -1) + : listObjectsChunkedAll(bucketName, null, null, -1); + StorageObject[] _objects = chunk == null ? null : chunk.getObjects(); + S3Info info; + + long validUntil = System.currentTimeMillis() + timeout; + int index; + _list = new ValidUntilMap(validUntil); + objects.put(key, _list); + + ArrayList commonPrefixes = new ArrayList(); + if (chunk.getCommonPrefixes().length > 0){ + commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); + for (String cp: commonPrefixes){ + info = new ParentObject(bucketName, cp, null, validUntil); + if (!hasObjName || cp.equals(nameFile) || cp.startsWith(nameDir)) _list.put(cp, info); + exists.put(toKey(bucketName, cp), info); + // add parent pseudo folders + while ((index = cp.lastIndexOf('/')) != -1) { + cp = cp.substring(0, index); + exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil)); + } + } + } + + String name; + StorageObjectWrapper tmp; + StorageObject stoObj = null; + // direct match + for (StorageObject kids: _objects) { + name = kids.getName(); + tmp = new StorageObjectWrapper(this, stoObj = kids, bucketName, validUntil); + + if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kids.getKey(), tmp); + exists.put(toKey(kids.getBucketName(), name), tmp); + + // add parent pseudo folders + while ((index = name.lastIndexOf('/')) != -1) { + name = name.substring(0, index); + exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil)); + } + } } return _list; } - catch (ServiceException se) { - throw toS3Exception(se); + catch (FactoryConfigurationError fce) { + XMLUtil.validateDocumentBuilderFactory(); + throw fce; } + catch (Exception e) { + e.printStackTrace(); + } + return null; } private String toKey(String bucketName, String objectName) { @@ -560,11 +612,12 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception long validUntil = System.currentTimeMillis() + timeout; StorageObject[] objects = chunk == null ? null : chunk.getObjects(); - - List commonPrefixes = new ArrayList(); - if (chunk.getCommonPrefixes().length > 0){ - //String[] cp = chunk.getCommonPrefixes(); - //aprint.o(cp.toString()); + String x = "1"; + if (objectName.indexOf("ss") > 0) + x = "2"; + + if (chunk !=null && chunk.getCommonPrefixes().length > 0){ + List commonPrefixes = new ArrayList(); commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); if (commonPrefixes.contains(nameDir)){ // pseudo directory @@ -576,35 +629,37 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it } - String targetName; - StorageObject stoObj = null; - // direct match - for (StorageObject so: objects) { - targetName = so.getName(); - if (nameFile.equals(targetName) || nameDir.equals(targetName)) { - exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil)); - } - } - - // pseudo directory? - if (info == null) { + if (objects != null){ + String targetName; + StorageObject stoObj = null; + // direct match for (StorageObject so: objects) { targetName = so.getName(); - if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) { - exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); + if (nameFile.equals(targetName) || nameDir.equals(targetName)) { + exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil)); } } - } - for (StorageObject obj: objects) { - if (stoObj != null && stoObj.equals(obj)) continue; - exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil)); - } + // pseudo directory? + if (info == null) { + for (StorageObject so: objects) { + targetName = so.getName(); + if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) { + exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); + } + } + } + + for (StorageObject obj: objects) { + if (stoObj != null && stoObj.equals(obj)) continue; + exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil)); + } - if (info == null) { - exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does - // not exis - ); + if (info == null) { + exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does + // not exis + ); + } } return info; } @@ -631,6 +686,19 @@ public StorageObjectsChunk listObjectsChunkedSilent(String bucketName, String ob return null; } + public StorageObjectsChunk listObjectsChunkedAll(String bucketName, String objectName, String delim, int max) { + try { + return getS3Service().listObjectsChunked(bucketName, objectName, delim, max, "", true); + } + catch (FactoryConfigurationError fce) { + XMLUtil.validateDocumentBuilderFactory(); + throw fce; + } + catch (Exception e) { + } + return null; + } + public S3BucketInfo get(String bucketName) throws S3Exception { bucketName = improveBucketName(bucketName); @@ -701,9 +769,17 @@ public void delete(String bucketName, String objectName, boolean force) throws S } ObjectKeyAndVersion[] keys = toObjectKeyAndVersions(list, null); - if (!force - && (keys.length > 1 || (keys.length == 1 && keys[0].getKey().length() > nameDir.length() && keys[0].getKey().substring(nameDir.length()).indexOf('/') != -1))) { - throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty"); + if (!force){ + String _key; + if (keys.length > 0) + _key = keys[0].getKey(); + + if (keys.length > 1 || (keys.length == 1 + && keys[0].getKey().length() > nameDir.length() + && keys[0].getKey().substring(nameDir.length()).indexOf('/') != -1) + ) { + throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty"); + } } // clear cache From 3468aafff60347a3d5e6294b38a755cc2ab3873f Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 4 Aug 2021 18:40:05 +0200 Subject: [PATCH 16/19] target java 1.8 --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index fa399fc..19cff3a 100644 --- a/build.xml +++ b/build.xml @@ -92,7 +92,7 @@ resource: "[{'class':'${class}','bundleName':'${bundlename}','bundleVersion':'${ - + From e855729febfd3f3c299812daea4336d2b3b1b361 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 4 Aug 2021 18:41:30 +0200 Subject: [PATCH 17/19] fix delete check when force is false --- .../org/lucee/extension/resource/s3/S3.java | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 92714db..9228df7 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -318,6 +318,7 @@ public List list(boolean recursive, boolean listPseudoFolder) throws S3E * @param recursive show all objects (recursive==true) or direct kids * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders * @param onlyChildren + * @param noCache * @return * @throws S3Exception * @@ -465,6 +466,12 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea int index; _list = new ValidUntilMap(validUntil); objects.put(key, _list); + + // add bucket + if (!hasObjName && !onlyChildren) { + S3Bucket b = getS3Service().getBucket(bucketName); + _list.put("", new S3BucketWrapper(b, validUntil)); + } ArrayList commonPrefixes = new ArrayList(); if (chunk.getCommonPrefixes().length > 0){ @@ -612,16 +619,19 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception long validUntil = System.currentTimeMillis() + timeout; StorageObject[] objects = chunk == null ? null : chunk.getObjects(); - String x = "1"; - if (objectName.indexOf("ss") > 0) - x = "2"; - + if (chunk !=null && chunk.getCommonPrefixes().length > 0){ - List commonPrefixes = new ArrayList(); + ArrayList commonPrefixes = new ArrayList(); + int index; commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); - if (commonPrefixes.contains(nameDir)){ - // pseudo directory - exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); + for (String cp: commonPrefixes){ + while ((index = cp.lastIndexOf('/')) != -1) { + cp = cp.substring(0, index); + if (cp.equals(nameFile) || cp.equals(nameDir)) + exists.put(toKey(bucketName, cp), info = new ParentObject(bucketName, cp, null, validUntil)); + else + exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil)); + } } } @@ -770,13 +780,10 @@ public void delete(String bucketName, String objectName, boolean force) throws S ObjectKeyAndVersion[] keys = toObjectKeyAndVersions(list, null); if (!force){ - String _key; - if (keys.length > 0) - _key = keys[0].getKey(); - + // TODO not sure of the logic here, why was it checking only for a sub directory? if (keys.length > 1 || (keys.length == 1 && keys[0].getKey().length() > nameDir.length() - && keys[0].getKey().substring(nameDir.length()).indexOf('/') != -1) + && keys[0].getKey().substring(nameDir.length()).length() > 0) //indexOf('/') != -1) ) { throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty"); } From b25830fe60972772d085611ae0e807053e7a147a Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 4 Aug 2021 19:26:59 +0200 Subject: [PATCH 18/19] fix merge conflict, remove whitespace --- .../lucee/extension/resource/s3/util/ApplicationSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java index d59e444..619d743 100644 --- a/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java +++ b/source/java/src/org/lucee/extension/resource/s3/util/ApplicationSettings.java @@ -66,7 +66,7 @@ public ApplicationSettings() { public static S3PropertiesCollection readS3PropertiesCollection(PageContext pc) throws PageException { Properties prop = pc.getApplicationContext().getS3(); String key = prop == null ? pc.getId() + ":" + pc.getStartTime() : "" + prop.hashCode(); - + S3PropertiesCollection existing = propsColl.get(key); if (existing != null) { return existing; From e8c6c6283c3db69e31b6818662bf9f7ffe2f63ba Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Thu, 5 Aug 2021 13:04:54 +0200 Subject: [PATCH 19/19] more null checks and logout host on error --- .../org/lucee/extension/resource/s3/S3.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 9228df7..e3c53d7 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -136,7 +136,7 @@ public S3Bucket createDirectory(String bucketName, AccessControlList acl, String } catch (ServiceException se) { throw toS3Exception(se, "could not create the bucket [" + bucketName - + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html"); + + "] at [" + host + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html"); } catch (FactoryConfigurationError fce) { XMLUtil.validateDocumentBuilderFactory(); @@ -474,7 +474,7 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea } ArrayList commonPrefixes = new ArrayList(); - if (chunk.getCommonPrefixes().length > 0){ + if (chunk != null && chunk.getCommonPrefixes().length > 0){ commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); for (String cp: commonPrefixes){ info = new ParentObject(bucketName, cp, null, validUntil); @@ -487,22 +487,23 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea } } } + if ( _objects != null ){ + String name; + StorageObjectWrapper tmp; + StorageObject stoObj = null; + // direct match + for (StorageObject kids: _objects) { + name = kids.getName(); + tmp = new StorageObjectWrapper(this, stoObj = kids, bucketName, validUntil); + + if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kids.getKey(), tmp); + exists.put(toKey(kids.getBucketName(), name), tmp); - String name; - StorageObjectWrapper tmp; - StorageObject stoObj = null; - // direct match - for (StorageObject kids: _objects) { - name = kids.getName(); - tmp = new StorageObjectWrapper(this, stoObj = kids, bucketName, validUntil); - - if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kids.getKey(), tmp); - exists.put(toKey(kids.getBucketName(), name), tmp); - - // add parent pseudo folders - while ((index = name.lastIndexOf('/')) != -1) { - name = name.substring(0, index); - exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil)); + // add parent pseudo folders + while ((index = name.lastIndexOf('/')) != -1) { + name = name.substring(0, index); + exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil)); + } } }