From 6d2cd737ccc407e8e0662c1439244e41a9c21bb6 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Wed, 22 Nov 2023 15:13:25 +0100 Subject: [PATCH] improve performance for listing with filter --- build.number | 4 +- .../extension/resource/ResourceSupport.java | 70 +++----- .../org/lucee/extension/resource/s3/S3.java | 170 ++++++++++-------- .../extension/resource/s3/S3Resource.java | 40 +++-- .../resource/s3/S3ResourceProvider.java | 8 +- .../resource/s3/S3ResourceS3InfoListener.java | 52 ++++++ .../resource/s3/listener/S3InfoListener.java | 11 ++ 7 files changed, 213 insertions(+), 142 deletions(-) create mode 100644 source/java/src/org/lucee/extension/resource/s3/S3ResourceS3InfoListener.java create mode 100644 source/java/src/org/lucee/extension/resource/s3/listener/S3InfoListener.java diff --git a/build.number b/build.number index b0bfe48..8f686e7 100644 --- a/build.number +++ b/build.number @@ -1,3 +1,3 @@ #Build Number for ANT. Do not edit! -#Thu Nov 16 14:21:37 CET 2023 -build.number=12 +#Wed Nov 22 15:12:11 CET 2023 +build.number=13 diff --git a/source/java/src/org/lucee/extension/resource/ResourceSupport.java b/source/java/src/org/lucee/extension/resource/ResourceSupport.java index e4208df..66def4e 100755 --- a/source/java/src/org/lucee/extension/resource/ResourceSupport.java +++ b/source/java/src/org/lucee/extension/resource/ResourceSupport.java @@ -20,8 +20,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; import lucee.commons.io.res.Resource; import lucee.commons.io.res.filter.ResourceFilter; @@ -108,71 +106,47 @@ private static void checkMoveToOK(Resource source, Resource target) throws IOExc @Override public String[] list(ResourceFilter filter) { - String[] files = list(); - if (files == null) return null; - List list = new ArrayList(); - Resource res; - for (int i = 0; i < files.length; i++) { - res = getRealResource(files[i]); - if (filter.accept(res)) list.add(files[i]); - } - return (String[]) list.toArray(new String[list.size()]); + return list(null, filter); } @Override public String[] list(ResourceNameFilter filter) { - String[] lst = list(); - if (lst == null) return null; + return list(filter, null); + } - List list = new ArrayList(); - for (int i = 0; i < lst.length; i++) { - if (filter.accept(getParentResource(), lst[i])) list.add(lst[i]); + @Override + public String[] list() { + return list(null, null); + } + + private String[] list(ResourceNameFilter nameilter, ResourceFilter filter) { + Resource[] children = listResources(nameilter, filter); + if (children == null) return null; + String[] rtn = new String[children.length]; + for (int i = 0; i < children.length; i++) { + rtn[i] = children[i].getName(); } - if (list.size() == 0) return new String[0]; - if (list.size() == lst.length) return lst; - return (String[]) list.toArray(new String[list.size()]); + return rtn; } @Override public Resource[] listResources(ResourceNameFilter filter) { - String[] files = list(); - if (files == null) return null; - - List list = new ArrayList(); - for (int i = 0; i < files.length; i++) { - if (filter.accept(this, files[i])) list.add(getRealResource(files[i])); - } - return (Resource[]) list.toArray(new Resource[list.size()]); + return listResources(filter, null); } @Override public Resource[] listResources(ResourceFilter filter) { - String[] files = list(); - if (files == null) return null; - - List list = new ArrayList(); - Resource res; - for (int i = 0; i < files.length; i++) { - res = this.getRealResource(files[i]); - if (filter.accept(res)) list.add(res); - } - return (Resource[]) list.toArray(new Resource[list.size()]); + return listResources(null, filter); } @Override - public String getReal(String realpath) { - return getRealResource(realpath).getPath(); + public Resource[] listResources() { + return listResources(null, null); } @Override - public String[] list() { - Resource[] children = listResources(); - if (children == null) return null; - String[] rtn = new String[children.length]; - for (int i = 0; i < children.length; i++) { - rtn[i] = children[i].getName(); - } - return rtn; + public String getReal(String realpath) { + return getRealResource(realpath).getPath(); } @Override @@ -326,4 +300,6 @@ private static String getCanonicalPathEL(Resource res) { return res.toString(); } } + + public abstract Resource[] listResources(ResourceNameFilter nameFilter, ResourceFilter filter); } \ No newline at end of file diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index b33c2a2..747b457 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -26,9 +26,12 @@ import org.lucee.extension.resource.s3.info.S3BucketWrapper; import org.lucee.extension.resource.s3.info.S3Info; import org.lucee.extension.resource.s3.info.StorageObjectWrapper; +import org.lucee.extension.resource.s3.listener.ListListener; +import org.lucee.extension.resource.s3.listener.S3InfoListener; import org.lucee.extension.resource.s3.region.RegionFactory; import org.lucee.extension.resource.s3.region.RegionFactory.Region; import org.lucee.extension.resource.s3.util.XMLUtil; +import org.lucee.extension.resource.s3.util.print; import com.amazonaws.AmazonServiceException; import com.amazonaws.HttpMethod; @@ -135,15 +138,14 @@ public static S3 getInstance(S3Properties props, long cache, Config config) { if (errorThreshold == 0) { synchronized (AWS) { if (errorThreshold == 0) { - errorThreshold = CFMLEngineFactory.getInstance().getCastUtil().toIntValue(S3Util.getSystemPropOrEnvVar("lucee.s3.listErrorThreshold", null), - 100000); + errorThreshold = S3ResourceProvider.toIntValue(S3Util.getSystemPropOrEnvVar("lucee.s3.listErrorThreshold", null), 100000); } } } if (warnThreshold == 0) { synchronized (AWS) { if (warnThreshold == 0) { - warnThreshold = CFMLEngineFactory.getInstance().getCastUtil().toIntValue(S3Util.getSystemPropOrEnvVar("lucee.s3.listWarnThreshold", null), 10000); + warnThreshold = S3ResourceProvider.toIntValue(S3Util.getSystemPropOrEnvVar("lucee.s3.listWarnThreshold", null), 10000); } } } @@ -167,13 +169,20 @@ public static S3 getInstance(S3Properties props, long cache, Config config) { } public static Log getLog(Config config) { - if (config == null) config = CFMLEngineFactory.getInstance().getThreadConfig(); if (_log == null || _log.getClass().getClassLoader() != config.getClass().getClassLoader()) { // does the logName exist? if (logName == null) { synchronized (DEFAULT_HOST) { if (logName == null) { + if (config == null) { + try { + config = CFMLEngineFactory.getInstance().getThreadConfig(); + } + catch (Exception e) { + + } + } if (config != null) { try { for (String ln: S3Util.getLogNames(config)) { @@ -690,7 +699,7 @@ public List list(boolean recursive, boolean listPseudoFolder) throws S3E info = it.next(); list.add(info); if (recursive) { - Iterator iit = list(info.getBucketName(), "", recursive, listPseudoFolder, true).iterator(); + Iterator iit = list(info.getBucketName(), "", recursive, listPseudoFolder, true, false, null).iterator(); while (iit.hasNext()) { list.add(iit.next()); } @@ -706,72 +715,26 @@ public List list(boolean recursive, boolean listPseudoFolder) throws S3E } } - /** - * list all allements in a specific bucket - * - * @param bucketName name of the bucket - * @param recursive show all objects (recursive==true) or direct kids - * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders - * @return - * @throws S3Exception - * - * public List list(String bucketName, boolean recursive, boolean - * listPseudoFolder) throws S3Exception { return list(bucketName, "", recursive, - * listPseudoFolder); } - */ - - public List list(String bucketName, String objectName, boolean recursive, boolean listPseudoFolder, boolean onlyChildren) throws S3Exception { - return list(bucketName, objectName, recursive, listPseudoFolder, onlyChildren, false); - } - - public List list(String bucketName, String objectName, boolean recursive, boolean listPseudoFolder, boolean onlyChildren, boolean noCache) throws S3Exception { - bucketName = improveBucketName(bucketName); - ValidUntilMap objects = _list(bucketName, objectName, onlyChildren, noCache); - - Iterator it = objects.values().iterator(); - Map map = new LinkedHashMap(); - S3Info info; - while (it.hasNext()) { - info = it.next(); - add(map, info, objectName, recursive, listPseudoFolder, onlyChildren); - } - Iterator iit = map.values().iterator(); - List list = new ArrayList(); - while (iit.hasNext()) { - list.add(iit.next()); - } - return list; - } - - /* - * public List listObjectSummaries(String bucketName) throws S3Exception { - * AmazonS3Client client = getAmazonS3(bucketName, null); try { ObjectListing objects = - * client.listObjects(bucketName); // Recursively delete all the objects inside given bucket - * List summeries = new ArrayList<>(); if (objects != null && - * objects.getObjectSummaries() != null) { while (true) { for (S3ObjectSummary summary: - * objects.getObjectSummaries()) { fixBackBlazeBug(summary, bucketName); summeries.add(summary); } - * if (objects.isTruncated()) { objects = client.listNextBatchOfObjects(objects); } else { break; } - * } } return summeries; - * - * } catch (AmazonServiceException ase) { throw toS3Exception(ase); } finally { client.release(); } - * } - */ - public List listObjects(String bucketName) throws S3Exception { AmazonS3Client client = getAmazonS3(bucketName, null); try { - ObjectListing objects = client.listObjects(bucketName); + + ListObjectsRequest lor = new ListObjectsRequest(); + lor.setBucketName(bucketName); + lor.setMaxKeys(2); + + ObjectListing objects = client.listObjects(lor); /* Recursively delete all the objects inside given bucket */ List list = new ArrayList<>(); int sum = 0; - if (objects != null) { - List summeries = objects.getObjectSummaries(); - if (summeries != null) { - sum += summeries.size(); - // we do it already here to avoid a OOME later on - if (sum >= warnThreshold) logThreshold(sum, bucketName); - while (true) { + while (true) { + if (objects != null) { + List summeries = objects.getObjectSummaries(); + if (summeries != null) { + sum += summeries.size(); + // we do it already here to avoid a OOME later on + if (sum >= warnThreshold) logThreshold(sum, bucketName); for (S3ObjectSummary summary: summeries) { fixBackBlazeBug(summary, bucketName); if (log != null) log.debug("S3", "get [" + bucketName + "/" + summary.getKey() + "]"); @@ -848,7 +811,7 @@ public Query listBucketsAsQuery() throws S3Exception, PageException { } } - public Query listObjectsAsQuery(String bucketName) throws S3Exception, PageException { + public Query listObjectsAsQuery(String bucketName, int maxKeys, ListListener listener) throws S3Exception, PageException { AmazonS3Client client = getAmazonS3(bucketName, null); try { CFMLEngine eng = CFMLEngineFactory.getInstance(); @@ -858,9 +821,14 @@ public Query listObjectsAsQuery(String bucketName) throws S3Exception, PageExcep final Key size = creator.createKey("size"); final Key lastModified = creator.createKey("lastModified"); final Key owner = creator.createKey("owner"); - Query qry = eng.getCreationUtil().createQuery(new Key[] { objectName, size, lastModified, owner }, 0, "buckets"); + Query qry = listener == null ? eng.getCreationUtil().createQuery(new Key[] { objectName, size, lastModified, owner }, 0, "buckets") : null; + ListObjectsRequest lor = new ListObjectsRequest(); + lor.setBucketName(bucketName); - ObjectListing objects = client.listObjects(bucketName); + if (maxKeys < 1 || maxKeys > 1000) throw new S3Exception("invalid maxKeys [" + maxKeys + "] must be an number between 1 and 1000"); + lor.setMaxKeys(maxKeys); + + ObjectListing objects = client.listObjects(lor); /* Recursively delete all the objects inside given bucket */ if (objects != null && objects.getObjectSummaries() != null) { @@ -869,6 +837,7 @@ public Query listObjectsAsQuery(String bucketName) throws S3Exception, PageExcep while (true) { summeries = objects.getObjectSummaries(); sum += summeries.size(); + if (listener != null) qry = eng.getCreationUtil().createQuery(new Key[] { objectName, size, lastModified, owner }, 0, "buckets"); if (sum >= warnThreshold) logThreshold(sum, bucketName); for (S3ObjectSummary summary: summeries) { fixBackBlazeBug(summary, bucketName); @@ -878,16 +847,20 @@ public Query listObjectsAsQuery(String bucketName) throws S3Exception, PageExcep qry.setAt(size, row, summary.getSize()); qry.setAt(owner, row, summary.getOwner().getDisplayName()); } + if (objects.isTruncated()) { + if (listener != null && !listener.invoke(qry)) break; objects = client.listNextBatchOfObjects(objects); + if (objects == null) break; } else { + if (listener != null) listener.invoke(qry); break; } } if (log != null) log.debug("S3", "list objects as query (" + sum + ") for [" + bucketName + "/" + "]"); } - return qry; + return listener == null ? qry : null; } catch (AmazonServiceException ase) { throw toS3Exception(ase); @@ -950,7 +923,23 @@ private static boolean isDirectKid(String name, String prefix) throws S3Exceptio return sub.indexOf('/') == -1; } - private ValidUntilMap _list(String bucketName, String objectName, boolean onlyChildren, boolean noCache) throws S3Exception { + /** + * list all allements in a specific bucket + * + * @param bucketName name of the bucket + * @param recursive show all objects (recursive==true) or direct kids + * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders + * @return + * @throws S3Exception + * + * public List list(String bucketName, boolean recursive, boolean + * listPseudoFolder) throws S3Exception { return list(bucketName, "", recursive, + * listPseudoFolder); } + */ + public List list(String bucketName, final String objectName, boolean recursive, boolean listPseudoFolder, boolean onlyChildren, boolean noCache, + S3InfoListener listener) throws S3Exception { + bucketName = improveBucketName(bucketName); + try { String key = toKey(bucketName, objectName); String nameDir = improveObjectName(objectName, true); @@ -958,11 +947,13 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea boolean hasObjName = !Util.isEmpty(objectName); // not cached - ValidUntilMap _list = cacheTimeout <= 0 || noCache ? null : cache.objects.get(key); + ValidUntilMap _list = cacheTimeout <= 0 || noCache || listener != null ? null : cache.objects.get(key); if (_list == null || _list.validUntil < System.currentTimeMillis()) { long validUntil = System.currentTimeMillis() + cacheTimeout; _list = new ValidUntilMap(validUntil); - cache.objects.put(key, _list); + + // we don't cache when listener is used because never all records are loaded + if (listener != null) cache.objects.put(key, _list); // add bucket if (!hasObjName && !onlyChildren) { @@ -985,7 +976,12 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea } } AmazonS3Client client = getAmazonS3(bucketName, null); - ObjectListing list = (hasObjName ? client.listObjects(bucketName, nameFile) : client.listObjects(bucketName)); + ListObjectsRequest lor = new ListObjectsRequest(); + lor.setBucketName(bucketName); + if (hasObjName) lor.setPrefix(nameFile); + // lor.setMaxKeys(100); + + ObjectListing list = client.listObjects(lor); try { if (list != null && list.getObjectSummaries() != null) { int sum = 0; @@ -1012,6 +1008,11 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea } } + if (listener != null) { + listener.invoke(improveResult(_list, bucketName, objectName, recursive, listPseudoFolder, onlyChildren, noCache)); + _list.clear(); + } + if (list.isTruncated()) { list = client.listNextBatchOfObjects(list); } @@ -1026,13 +1027,36 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea client.release(); } } - return _list; + if (listener != null) return null; + return improveResult(_list, bucketName, objectName, recursive, listPseudoFolder, onlyChildren, noCache); } catch (AmazonS3Exception ase) { throw toS3Exception(ase); } } + private List improveResult(ValidUntilMap objects, String bucketName, String objectName, boolean recursive, boolean listPseudoFolder, boolean onlyChildren, + boolean noCache) throws S3Exception { + bucketName = improveBucketName(bucketName); + print.e("--- improveResult ----"); + print.e("- " + objects.size()); + Iterator it = objects.values().iterator(); + Map map = new LinkedHashMap(); + S3Info info; + while (it.hasNext()) { + info = it.next(); + add(map, info, objectName, recursive, listPseudoFolder, onlyChildren); + } + print.e("- " + map.size()); + Iterator iit = map.values().iterator(); + List list = new ArrayList(); + while (iit.hasNext()) { + list.add(iit.next()); + } + print.e("- " + list.size()); + return list; + } + private S3ObjectSummary fixBackBlazeBug(S3ObjectSummary kid, String bucketName) { // the method getBucketName does not deliver the bucket name if (kid != null && Util.isEmpty(kid.getBucketName()) && !Util.isEmpty(bucketName)) kid.setBucketName(bucketName); diff --git a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java index 7389fab..a5c4f8f 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3Resource.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3Resource.java @@ -30,6 +30,8 @@ import lucee.commons.io.res.Resource; import lucee.commons.io.res.ResourceProvider; +import lucee.commons.io.res.filter.ResourceFilter; +import lucee.commons.io.res.filter.ResourceNameFilter; import lucee.loader.engine.CFMLEngine; import lucee.loader.engine.CFMLEngineFactory; import lucee.loader.util.Util; @@ -52,7 +54,7 @@ public final class S3Resource extends ResourceSupport { private String location = null; private Object acl;// ="public-read"; - private S3Resource(CFMLEngine engine, S3 s3, S3Properties props, String location, S3ResourceProvider provider, String buckedName, String objectName) { + S3Resource(CFMLEngine engine, S3 s3, S3Properties props, String location, S3ResourceProvider provider, String buckedName, String objectName) { super(engine); this.s3 = s3; this.props = props; @@ -367,30 +369,36 @@ private S3Info getInfo() { } @Override - public Resource[] listResources() { + public Resource[] listResources(ResourceNameFilter nameFilter, ResourceFilter filter) { S3Resource[] children = null; + S3ResourceS3InfoListener listener = null; // long timeout=System.currentTimeMillis()+provider.getCache(); try { boolean buckets = false; - List list = null; if (isRoot()) { buckets = true; - list = s3.list(false, false); + List list = s3.list(false, false); + if (list != null) { + Iterator it = list.iterator(); + children = new S3Resource[list.size()]; + S3Info si; + int index = 0; + while (it.hasNext()) { + si = it.next(); + children[index] = new S3Resource(engine, s3, props, location, provider, si.getBucketName(), buckets ? "" : S3.improveObjectName(si.getObjectName(), false)); + index++; + } + } } else if (isDirectory()) { - list = isBucket() ? s3.list(bucketName, "", false, true, true) : s3.list(bucketName, objectName + "/", false, true, true); - } - - if (list != null) { - Iterator it = list.iterator(); - children = new S3Resource[list.size()]; - S3Info si; - int index = 0; - while (it.hasNext()) { - si = it.next(); - children[index] = new S3Resource(engine, s3, props, location, provider, si.getBucketName(), buckets ? "" : S3.improveObjectName(si.getObjectName(), false)); - index++; + listener = new S3ResourceS3InfoListener(provider, s3, props, location, nameFilter, filter); + if (isBucket()) { + s3.list(bucketName, "", false, true, true, false, listener); + } + else { + s3.list(bucketName, objectName + "/", false, true, true, false, listener); } + children = listener.getRecords(); } } catch (S3Exception e) { diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java index ee312ef..9602fbf 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java @@ -75,13 +75,13 @@ public ResourceProvider init(String scheme, Map arguments) { return this; } - private int toIntValue(String str, int defaultValue) { + public static int toIntValue(String str, int defaultValue) { + if (Util.isEmpty(str)) return defaultValue; try { return Integer.parseInt(str); } - catch (Throwable t) { - if (t instanceof ThreadDeath) throw (ThreadDeath) t; - return defaultValue; + catch (Exception e) { + return CFMLEngineFactory.getInstance().getCastUtil().toIntValue(str, defaultValue); } } diff --git a/source/java/src/org/lucee/extension/resource/s3/S3ResourceS3InfoListener.java b/source/java/src/org/lucee/extension/resource/s3/S3ResourceS3InfoListener.java new file mode 100644 index 0000000..7a4f82c --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/S3ResourceS3InfoListener.java @@ -0,0 +1,52 @@ +package org.lucee.extension.resource.s3; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.lucee.extension.resource.s3.info.S3Info; +import org.lucee.extension.resource.s3.listener.S3InfoListener; +import org.lucee.extension.resource.s3.util.print; + +import lucee.commons.io.res.filter.ResourceFilter; +import lucee.commons.io.res.filter.ResourceNameFilter; +import lucee.loader.engine.CFMLEngine; +import lucee.loader.engine.CFMLEngineFactory; + +public class S3ResourceS3InfoListener implements S3InfoListener { + public Map records = new LinkedHashMap<>(); + private CFMLEngine engine; + private S3ResourceProvider provider; + private S3 s3; + private S3Properties props; + private String location; + private ResourceNameFilter nameFilter; + private ResourceFilter filter; + + public S3ResourceS3InfoListener(S3ResourceProvider provider, S3 s3, S3Properties props, String location, ResourceNameFilter nameFilter, ResourceFilter filter) { + print.e("S3ResourceS3InfoListener:init"); + engine = CFMLEngineFactory.getInstance(); + this.provider = provider; + this.s3 = s3; + this.props = props; + this.location = location; + this.nameFilter = nameFilter; + this.filter = filter; + + } + + @Override + public void invoke(List infos) { + for (S3Info info: infos) { + S3Resource res = new S3Resource(engine, s3, props, location, provider, info.getBucketName(), S3.improveObjectName(info.getObjectName(), false)); + if (nameFilter != null && !nameFilter.accept(res.getParentResource(), res.getName())) continue; + if (filter != null && !filter.accept(res)) continue; + records.put(res.getAbsolutePath(), res); + } + } + + public S3Resource[] getRecords() { + return records.values().toArray(new S3Resource[records.size()]); + } + +} diff --git a/source/java/src/org/lucee/extension/resource/s3/listener/S3InfoListener.java b/source/java/src/org/lucee/extension/resource/s3/listener/S3InfoListener.java new file mode 100644 index 0000000..68b5f34 --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/listener/S3InfoListener.java @@ -0,0 +1,11 @@ +package org.lucee.extension.resource.s3.listener; + +import java.util.List; + +import org.lucee.extension.resource.s3.info.S3Info; + +public interface S3InfoListener { + + void invoke(List improveResult); + +}