diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 46e1fbc..362f951 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,9 @@ name: Java CI -on: [push, pull_request,workflow_dispatch] +on: [push, pull_request, workflow_dispatch] + +concurrency: s3_extension jobs: build: @@ -61,7 +63,10 @@ jobs: testLabels: s3 testAdditional: ${{ github.workspace }}/tests S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_ID_TEST }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }} + SS3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }} S3_CUSTOM_ACCESS_KEY_ID: "minioadmin" S3_CUSTOM_SECRET_KEY: "minioadmin" - S3_CUSTOM_HOST: "localhost:9000" + S3_CUSTOM_HOST: "http://localhost:9000" + S3_GOOGLE_ACCESS_KEY_ID: ${{ secrets.S3_GOOGLE_SECRET_KEY }} + SS3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }} + S3_GOOGLE_HOST: "storage.googleapis.com" diff --git a/build.xml b/build.xml index fa399fc..19cff3a 100644 --- a/build.xml +++ b/build.xml @@ -92,7 +92,7 @@ resource: "[{'class':'${class}','bundleName':'${bundlename}','bundleVersion':'${ - + diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 305c502..e3c53d7 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -136,7 +136,7 @@ public S3Bucket createDirectory(String bucketName, AccessControlList acl, String } catch (ServiceException se) { throw toS3Exception(se, "could not create the bucket [" + bucketName - + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html"); + + "] at [" + host + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html"); } catch (FactoryConfigurationError fce) { XMLUtil.validateDocumentBuilderFactory(); @@ -314,8 +314,11 @@ public List list(boolean recursive, boolean listPseudoFolder) throws S3E * list all allements in a specific bucket * * @param bucketName name of the bucket + * @param objectName name of the object * @param recursive show all objects (recursive==true) or direct kids * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders + * @param onlyChildren + * @param noCache * @return * @throws S3Exception * @@ -423,8 +426,8 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea // not cached ValidUntilMap _list = timeout <= 0 || noCache ? null : objects.get(key); if (_list == null || _list.validUntil < System.currentTimeMillis()) { + /* S3Object[] kids = hasObjName ? getS3Service().listObjects(bucketName, nameFile, ",") : getS3Service().listObjects(bucketName); - long validUntil = System.currentTimeMillis() + timeout; _list = new ValidUntilMap(validUntil); objects.put(key, _list); @@ -450,13 +453,71 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil)); } } + */ + + /* ------------------------ chunked, which includes prefixes ------------------------------ */ + StorageObjectsChunk chunk = hasObjName ? + listObjectsChunkedAll(bucketName, nameFile, ",", -1) + : listObjectsChunkedAll(bucketName, null, null, -1); + StorageObject[] _objects = chunk == null ? null : chunk.getObjects(); + S3Info info; + + long validUntil = System.currentTimeMillis() + timeout; + int index; + _list = new ValidUntilMap(validUntil); + objects.put(key, _list); + + // add bucket + if (!hasObjName && !onlyChildren) { + S3Bucket b = getS3Service().getBucket(bucketName); + _list.put("", new S3BucketWrapper(b, validUntil)); + } + + ArrayList commonPrefixes = new ArrayList(); + if (chunk != null && chunk.getCommonPrefixes().length > 0){ + commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); + for (String cp: commonPrefixes){ + info = new ParentObject(bucketName, cp, null, validUntil); + if (!hasObjName || cp.equals(nameFile) || cp.startsWith(nameDir)) _list.put(cp, info); + exists.put(toKey(bucketName, cp), info); + // add parent pseudo folders + while ((index = cp.lastIndexOf('/')) != -1) { + cp = cp.substring(0, index); + exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil)); + } + } + } + if ( _objects != null ){ + String name; + StorageObjectWrapper tmp; + StorageObject stoObj = null; + // direct match + for (StorageObject kids: _objects) { + name = kids.getName(); + tmp = new StorageObjectWrapper(this, stoObj = kids, bucketName, validUntil); + + if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kids.getKey(), tmp); + exists.put(toKey(kids.getBucketName(), name), tmp); + + // add parent pseudo folders + while ((index = name.lastIndexOf('/')) != -1) { + name = name.substring(0, index); + exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil)); + } + } + } } return _list; } - catch (ServiceException se) { - throw toS3Exception(se); + catch (FactoryConfigurationError fce) { + XMLUtil.validateDocumentBuilderFactory(); + throw fce; + } + catch (Exception e) { + e.printStackTrace(); } + return null; } private String toKey(String bucketName, String objectName) { @@ -560,41 +621,56 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception long validUntil = System.currentTimeMillis() + timeout; StorageObject[] objects = chunk == null ? null : chunk.getObjects(); - if (objects == null || objects.length == 0) { - exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it - // does - return null; + if (chunk !=null && chunk.getCommonPrefixes().length > 0){ + ArrayList commonPrefixes = new ArrayList(); + int index; + commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); + for (String cp: commonPrefixes){ + while ((index = cp.lastIndexOf('/')) != -1) { + cp = cp.substring(0, index); + if (cp.equals(nameFile) || cp.equals(nameDir)) + exists.put(toKey(bucketName, cp), info = new ParentObject(bucketName, cp, null, validUntil)); + else + exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil)); + } + } } - String targetName; - StorageObject stoObj = null; - // direct match - for (StorageObject so: objects) { - targetName = so.getName(); - if (nameFile.equals(targetName) || nameDir.equals(targetName)) { - exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil)); - } + if (info == null && (objects == null || objects.length == 0)) { + exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it } - // pseudo directory? - if (info == null) { + if (objects != null){ + String targetName; + StorageObject stoObj = null; + // direct match for (StorageObject so: objects) { targetName = so.getName(); - if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) { - exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); + if (nameFile.equals(targetName) || nameDir.equals(targetName)) { + exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil)); } } - } - for (StorageObject obj: objects) { - if (stoObj != null && stoObj.equals(obj)) continue; - exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil)); - } + // pseudo directory? + if (info == null) { + for (StorageObject so: objects) { + targetName = so.getName(); + if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) { + exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil)); + } + } + } + + for (StorageObject obj: objects) { + if (stoObj != null && stoObj.equals(obj)) continue; + exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil)); + } - if (info == null) { - exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does - // not exis - ); + if (info == null) { + exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does + // not exis + ); + } } return info; } @@ -621,6 +697,19 @@ public StorageObjectsChunk listObjectsChunkedSilent(String bucketName, String ob return null; } + public StorageObjectsChunk listObjectsChunkedAll(String bucketName, String objectName, String delim, int max) { + try { + return getS3Service().listObjectsChunked(bucketName, objectName, delim, max, "", true); + } + catch (FactoryConfigurationError fce) { + XMLUtil.validateDocumentBuilderFactory(); + throw fce; + } + catch (Exception e) { + } + return null; + } + public S3BucketInfo get(String bucketName) throws S3Exception { bucketName = improveBucketName(bucketName); @@ -691,9 +780,14 @@ public void delete(String bucketName, String objectName, boolean force) throws S } ObjectKeyAndVersion[] keys = toObjectKeyAndVersions(list, null); - if (!force - && (keys.length > 1 || (keys.length == 1 && keys[0].getKey().length() > nameDir.length() && keys[0].getKey().substring(nameDir.length()).indexOf('/') != -1))) { - throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty"); + if (!force){ + // TODO not sure of the logic here, why was it checking only for a sub directory? + if (keys.length > 1 || (keys.length == 1 + && keys[0].getKey().length() > nameDir.length() + && keys[0].getKey().substring(nameDir.length()).length() > 0) //indexOf('/') != -1) + ) { + throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty"); + } } // clear cache @@ -1278,9 +1372,27 @@ private S3Service getS3Service() { synchronized (getToken(accessKeyId + ":" + secretAccessKey)) { if (service == null) { final Jets3tProperties props = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME); - if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) { - props.setProperty("s3service.s3-endpoint", host); - } + props.clearAllProperties(); + //if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) { + int hasPort = host.indexOf(':'); + if (hasPort == -1) { + props.setProperty("s3service.s3-endpoint", host); // no port, no protocol + } else { + int hasHttp = host.toLowerCase().indexOf("http", 0); // http(s)://localhost:9000 prefix? + if (hasHttp == -1){ // no http prefix + props.setProperty("s3service.s3-endpoint", host.substring(0, hasPort)); + props.setProperty("s3service.s3-endpoint-http-port", host.substring(hasPort + 1)); + } else { // with http(s) prefix + String protocol = host.substring(0,host.indexOf(":")); + props.setProperty("s3service.https-only", protocol.equalsIgnoreCase("http") ? "false" : "true" ); + String _host = host.substring(host.lastIndexOf("/") + 1); + hasPort = _host.indexOf(':'); + props.setProperty("s3service.s3-endpoint", _host.substring(0, hasPort)); + props.setProperty("s3service.s3-endpoint-http-port", _host.substring(hasPort + 1)); + props.setProperty("s3service.disable-dns-buckets", "true"); + } + } + //} service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props); } } diff --git a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java index 9b8ace2..187d838 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java @@ -182,7 +182,11 @@ else if (!Util.isEmpty(s3.getMappingName(), true)) { sb.append(s3.getMappingName()).append("@"); } - if (doHost) sb.append(s3.getHost()); + if (doHost && s3.getCustomCredentials()){ + if (sb.substring(sb.length()-1) != "@") + sb.append("@"); + sb.append(s3.getHost()); + } return sb.toString(); } diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index 1e3273a..fdf076c 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -167,9 +167,9 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, storage.setValue(defaultLocation); int atIndex = path.indexOf('@'); - int slashIndex = path.indexOf('/'); + int slashIndex = path.indexOf('/', atIndex); // secret keys can contain / if (slashIndex == -1) { - slashIndex = path.length(); + //slashIndex = path.length(); // never used! path += "/"; } int index; @@ -215,23 +215,35 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties, // } } - path = prettifyPath(path.substring(atIndex + 1)); - index = path.indexOf('/'); - properties.setHost(host); - if (index == -1) { - if (path.equalsIgnoreCase(S3.DEFAULT_HOST) || path.equalsIgnoreCase(host)) { - properties.setHost(path); - path = "/"; - } - } - else { - String _host = path.substring(0, index); - if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) { - properties.setHost(_host); - path = path.substring(index); + String srcPath = path; + if (atIndex == -1){ + // no host information, skip parsing, ie. s3://bucketname/ etc + path = prettifyPath(path); + } else if ( (slashIndex - atIndex) == 1 ){ + // key/secret@/bucket + path = prettifyPath(path.substring(atIndex + 1)); + } else { + int doubleIndex = path.indexOf("://", atIndex); // do we have a http(s):// + if (doubleIndex > 0) { + // key/secret@http://localhost:9000/bucket + index = path.indexOf('/', doubleIndex+3 ); + host = path.substring(atIndex+1, index); + path = prettifyPath(path.substring(index)); + } else { + path = prettifyPath(path.substring(atIndex + 1)); + index = path.indexOf('/'); + if (index == -1) { + path = "/"; + } else if ( (slashIndex - atIndex) != 1 ) { + // key/secret@storage.googleapis.com/bucket + host = path.substring(0, index); + path = path.substring(index); + } } } - + if (host.endsWith("/")) + host = host.substring(0, host.length() - 1); // strip trailing / after hostname with http://localhost:9000/ + properties.setHost(host); // get from system.properties/env.var if (Util.isEmpty(accessKeyId, true)) { if (Util.isEmpty(mappingName)) { @@ -350,4 +362,4 @@ public Map getArguments() { public char getSeparator() { return '/'; } -} \ No newline at end of file +}