diff --git a/build.number b/build.number index 8f686e7..e29e6bc 100644 --- a/build.number +++ b/build.number @@ -1,3 +1,3 @@ #Build Number for ANT. Do not edit! -#Wed Nov 22 15:12:11 CET 2023 -build.number=13 +#Wed Nov 22 15:37:52 CET 2023 +build.number=14 diff --git a/source/fld/function.fld b/source/fld/function.fld index 822ea17..9939977 100755 --- a/source/fld/function.fld +++ b/source/fld/function.fld @@ -989,6 +989,35 @@ Yes Name of the bucket to list objects from. + + + listener + closure,udf,function,component + any + No + true + Optional. Defines the target where the downloaded data will be directed. + If a file path is provided, the data is saved to that path, the file path must be provided with help of the function "fileOpen" like this [fileOpen(path,"write")]. + If a closure or function is given, it will be invoked with parts of the downloaded data as its argument. + The function should accept a single argument named 'line' for line-by-line processing, + 'string{Number}' for string blocks of a specified size, + or 'binary{Number}' for binary blocks of a specified size. + + The function should return a boolean value: returning false will stop further reading from S3, + while true will continue the process. + + If this argument is omitted, the function returns the downloaded data directly. + + + + + blockfactor + maxKeys,blockSize + numeric + No + 1000 + + accessKeyId accessKey,awsAccessKeyId,awsAccessKey diff --git a/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java b/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java index 71c05f1..e729730 100644 --- a/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java +++ b/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java @@ -69,13 +69,12 @@ public Object invoke(PageContext pc, Object[] args) throws PageException { if (target instanceof UDF) { targetUDF = (UDF) target; validateInvoke(pc, targetUDF, mode, blockSize, false); - } else if (target instanceof Component) { targetCFC = (Component) target; Component csa = toComponentSpecificAccess(Component.ACCESS_PRIVATE, targetCFC); - boolean hasBefore = toFunction(csa.get(BEFORE), null) != null; - boolean hasAfter = toFunction(csa.get(AFTER), null) != null; + boolean hasBefore = toFunction(csa.get(BEFORE, null), null) != null; + boolean hasAfter = toFunction(csa.get(AFTER, null), null) != null; UDF invoke = toFunction(csa.get(INVOKE), null); if (invoke == null) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 2, "component", "the listener component does not contain a instance function with name [invoke] that is required", null); @@ -218,7 +217,7 @@ else if (targetRes != null) { } } - private UDF toFunction(Object obj, UDF defaultValue) { + public static UDF toFunction(Object obj, UDF defaultValue) { if (obj instanceof UDF) return (UDF) obj; return defaultValue; } diff --git a/source/java/src/org/lucee/extension/resource/s3/function/S3ListBucket.java b/source/java/src/org/lucee/extension/resource/s3/function/S3ListBucket.java index 6520346..6046ef3 100644 --- a/source/java/src/org/lucee/extension/resource/s3/function/S3ListBucket.java +++ b/source/java/src/org/lucee/extension/resource/s3/function/S3ListBucket.java @@ -1,45 +1,69 @@ package org.lucee.extension.resource.s3.function; import org.lucee.extension.resource.s3.S3; +import org.lucee.extension.resource.s3.listener.ComponentListListener; +import org.lucee.extension.resource.s3.listener.ListListener; +import org.lucee.extension.resource.s3.listener.UDFListListener; import lucee.loader.engine.CFMLEngine; import lucee.loader.engine.CFMLEngineFactory; +import lucee.runtime.Component; import lucee.runtime.PageContext; import lucee.runtime.exp.PageException; -import lucee.runtime.type.Query; +import lucee.runtime.type.UDF; import lucee.runtime.util.Cast; public class S3ListBucket extends S3Function { private static final long serialVersionUID = 3486553628255584848L; - public static Query call(PageContext pc, String bucketName, String accessKeyId, String secretAccessKey, String host, double timeout) throws PageException { - + @Override + public Object invoke(PageContext pc, Object[] args) throws PageException { CFMLEngine eng = CFMLEngineFactory.getInstance(); - // for backward compatibility, when host was not existing - if (eng.getDecisionUtil().isNumber(host)) { - timeout = eng.getCastUtil().toDoubleValue(host); - host = null; - } + Cast cast = eng.getCastUtil(); + if (args.length < 1 || args.length > 7) throw eng.getExceptionUtil().createFunctionException(pc, "S3ListBucket", 1, 7, args.length); + + // required + String bucketName = cast.toString(args[0]); + + // optional + Object listener = args.length > 1 && args[1] != null ? args[1] : null; + int blockSize = args.length > 2 && args[2] != null ? cast.toIntValue(args[2]) : 1000; + String accessKeyId = args.length > 3 && args[3] != null ? cast.toString(args[3]) : null; + String secretAccessKey = args.length > 4 && args[4] != null ? cast.toString(args[4]) : null; + String host = args.length > 5 && args[5] != null ? cast.toString(args[5]) : null; + double timeout = args.length > 6 && !isEmpty(args[6]) ? cast.toDoubleValue(args[6]) : 0; + + // validate + try { S3 s3 = S3.getInstance(toS3Properties(pc, accessKeyId, secretAccessKey, host), toTimeout(timeout), pc.getConfig()); - return s3.listObjectsAsQuery(bucketName); + // no listener + if (listener == null) { + return s3.listObjectsAsQuery(bucketName, blockSize, null); + } + else { + ListListener list = toListener(eng, pc, listener); + list.before(); + s3.listObjectsAsQuery(bucketName, blockSize, toListener(eng, pc, listener)); + list.after(); + } } catch (Exception e) { throw CFMLEngineFactory.getInstance().getCastUtil().toPageException(e); } + return null; } - @Override - public Object invoke(PageContext pc, Object[] args) throws PageException { - CFMLEngine engine = CFMLEngineFactory.getInstance(); - Cast cast = engine.getCastUtil(); - if (args.length == 5) return call(pc, cast.toString(args[0]), cast.toString(args[1]), cast.toString(args[2]), cast.toString(args[3]), cast.toDoubleValue(args[4])); - if (args.length == 4) return call(pc, cast.toString(args[0]), cast.toString(args[1]), cast.toString(args[2]), cast.toString(args[3]), 0); - if (args.length == 3) return call(pc, cast.toString(args[0]), cast.toString(args[1]), cast.toString(args[2]), null, 0); - if (args.length == 2) return call(pc, cast.toString(args[0]), cast.toString(args[1]), null, null, 0); - if (args.length == 1) return call(pc, cast.toString(args[0]), null, null, null, 0); - throw engine.getExceptionUtil().createFunctionException(pc, "S3ListBucket", 1, 5, args.length); + private ListListener toListener(CFMLEngine eng, PageContext pc, Object listener) throws PageException { + if (listener instanceof UDF) { + return new UDFListListener(eng, pc, (UDF) listener); + } + if (listener instanceof Component) { + return new ComponentListListener(eng, pc, (Component) listener); + } + throw CFMLEngineFactory.getInstance().getExceptionUtil().createFunctionException(pc, "S3ListBucket", 2, "listener", + "invalid listener type [" + listener.getClass().getName() + "], only functions and components are supported as listeners", ""); } } \ No newline at end of file diff --git a/source/java/src/org/lucee/extension/resource/s3/listener/ComponentListListener.java b/source/java/src/org/lucee/extension/resource/s3/listener/ComponentListListener.java new file mode 100644 index 0000000..dfe4e88 --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/listener/ComponentListListener.java @@ -0,0 +1,61 @@ +package org.lucee.extension.resource.s3.listener; + +import org.lucee.extension.resource.s3.function.S3Download; + +import lucee.loader.engine.CFMLEngine; +import lucee.loader.util.Util; +import lucee.runtime.Component; +import lucee.runtime.PageContext; +import lucee.runtime.exp.PageException; +import lucee.runtime.type.Collection.Key; +import lucee.runtime.type.Query; + +public class ComponentListListener implements ListListener { + + private PageContext pc; + private Component listener; + private Component csa; + private Key INVOKE; + private Key BEFORE; + private Key AFTER; + private CFMLEngine eng; + + public ComponentListListener(CFMLEngine eng, PageContext pc, Component listener) throws PageException { + INVOKE = eng.getCastUtil().toKey("invoke"); + BEFORE = eng.getCastUtil().toKey("before"); + AFTER = eng.getCastUtil().toKey("after"); + this.eng = eng; + this.pc = pc; + this.listener = listener; + csa = S3Download.toComponentSpecificAccess(Component.ACCESS_PRIVATE, listener); + + } + + @Override + public void before() throws PageException { + if (S3Download.toFunction(csa.get(BEFORE, null), null) != null) { + listener.call(pc, BEFORE, new Object[] {}); + } + } + + @Override + public boolean invoke(Query data) throws PageException { + if (S3Download.toFunction(csa.get(INVOKE, null), null) != null) { + Object res = listener.call(pc, INVOKE, new Object[] { data }); + if (res == null || Util.isEmpty(res.toString())) return true; + return eng.getCastUtil().toBooleanValue(res); + } + else { + throw eng.getExceptionUtil().createFunctionException(pc, "S3ListBucket", 2, "component", + "the listener component does not contain a instance function with name [invoke] that is required", null); + } + } + + @Override + public void after() throws PageException { + if (S3Download.toFunction(csa.get(AFTER, null), null) != null) { + listener.call(pc, AFTER, new Object[] {}); + } + } + +} diff --git a/source/java/src/org/lucee/extension/resource/s3/listener/ListListener.java b/source/java/src/org/lucee/extension/resource/s3/listener/ListListener.java new file mode 100644 index 0000000..3bd4a44 --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/listener/ListListener.java @@ -0,0 +1,14 @@ +package org.lucee.extension.resource.s3.listener; + +import lucee.runtime.exp.PageException; +import lucee.runtime.type.Query; + +public interface ListListener { + + public void before() throws PageException; + + public boolean invoke(Query data) throws PageException; + + public void after() throws PageException; + +} diff --git a/source/java/src/org/lucee/extension/resource/s3/listener/UDFListListener.java b/source/java/src/org/lucee/extension/resource/s3/listener/UDFListListener.java new file mode 100644 index 0000000..ee0cce7 --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/listener/UDFListListener.java @@ -0,0 +1,37 @@ +package org.lucee.extension.resource.s3.listener; + +import lucee.loader.engine.CFMLEngine; +import lucee.loader.util.Util; +import lucee.runtime.PageContext; +import lucee.runtime.exp.PageException; +import lucee.runtime.type.Query; +import lucee.runtime.type.UDF; + +public class UDFListListener implements ListListener { + + private PageContext pc; + private UDF listener; + private CFMLEngine eng; + + public UDFListListener(CFMLEngine eng, PageContext pc, UDF listener) { + this.eng = eng; + this.pc = pc; + this.listener = listener; + } + + @Override + public boolean invoke(Query data) throws PageException { + Object res = listener.call(pc, new Object[] { data }, true); + if (res == null || Util.isEmpty(res.toString())) return true; + return eng.getCastUtil().toBooleanValue(res); + } + + @Override + public void before() { + } + + @Override + public void after() { + } + +} diff --git a/tests/functions/S3ListBucket.cfc b/tests/functions/S3ListBucket.cfc index f9f1287..fcdbb9f 100644 --- a/tests/functions/S3ListBucket.cfc +++ b/tests/functions/S3ListBucket.cfc @@ -3,18 +3,22 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="s3" { describe( title="Test suite for S3ListBucket()", body=function() { it(title="check region with blackbaze",skip=Util::isBackBlazeNotSupported(), body = function( currentSpec ) { testit(Util::getBackBlazeCredentials()); + testUDF(Util::getBackBlazeCredentials()); }); it(title="check with amazon",skip=Util::isAWSNotSupported(), body = function( currentSpec ) { testit(Util::getAWSCredentials()); + testUDF(Util::getAWSCredentials()); }); it(title="check with wasabi",skip=Util::isWasabiNotSupported(), body = function( currentSpec ) { testit(Util::getWasabiCredentials()); + testUDF(Util::getWasabiCredentials()); }); it(title="check with google",skip=Util::isGoogleNotSupported(), body = function( currentSpec ) { testit(Util::getGoogleCredentials()); + testUDF(Util::getGoogleCredentials()); }); }); @@ -51,6 +55,56 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="s3" { } } + private function testUDF(cred) { + try { + // create variables + var bucketName=cred.PREFIX&"-list-bucket:"&listFirst(replace(server.lucee.version,".","","all"),"-"); + var objectNames=["sub/test1.txt","sub/test2.txt","sub/test3.txt","sub/test4.txt","sub/test5.txt"]; + + // create empty bucket + Util::deleteIfExists(cred,bucketName); + S3CreateBucket( + bucketName:bucketName, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)); + + // create objects + loop array=objectNames item="local.objectName" { + // create source bucket + if(!S3Exists( + bucketName:bucketName, objectName:objectName, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST))) { + S3Write( + value:"Susi Sorglos", + bucketName:bucketName, objectName:objectName, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)); + } + } + + var res=""; + S3ListBucket( + bucketName:bucketName, + listener:function (data){ + res&=data.recordcount&";"; + dump(arguments); + }, + blockfactor:3, + accessKeyId:cred.ACCESS_KEY_ID, + secretAccessKey:cred.SECRET_KEY, + host:(isNull(cred.HOST)?nullvalue():cred.HOST) + ); + assertEquals("3;2;", res); + } + catch(e) { + if(!findNoCase("Transaction cap exceeded", e.message) ) throw e; + } + finally { + Util::deleteBucketEL(cred,bucketName); + } + } + + + + private function doFind(value){ return value EQ "world";