diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bb16b98..6f93c6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,10 +20,26 @@ jobs: DB_PASSWORD: root AWS_DOMAIN: amazonaws.com AWS_REGION: us-east-1 + continue-on-error: ${{ matrix.experimental }} strategy: fail-fast: false matrix: - cfengine: [ "lucee@5", "adobe@2018", "adobe@2021" ] + cfengine: [ "lucee@5", "adobe@2018", "adobe@2021" ] + coldboxVersion: [ "^6.0.0" ] + experimental: [ false ] + include: + - cfengine: "adobe@2023" + coldboxVersion: "^6.0.0" + experimental: true + - coldboxVersion: "be" + cfengine: "lucee@5" + experimental: true + - coldboxVersion: "be" + cfengine: "adobe@2018" + experimental: true + - coldboxVersion: "be" + cfengine: "adobe@2021" + experimental: true steps: - name: Checkout Repository uses: actions/checkout@v2 @@ -53,21 +69,22 @@ jobs: printf "AWS_REGION=${{ env.AWS_REGION }}\n" >> .env printf "AWS_DOMAIN=${{ env.AWS_DOMAIN }}\n" >> .env printf "ENGINE=${{ matrix.cfengine }}\n" >> .env + printf "COLDBOX_VERSION=${{ matrix.coldboxVersion }}\n" >> .env + - name: Setup CommandBox CLI uses: Ortus-Solutions/setup-commandbox@v2.0.1 - - name: Install Dependencies + - name: Install Test Harness with ColdBox ${{ matrix.coldboxVersion }} run: | - box run-script install:dependencies + box install + cd test-harness + box package set dependencies.coldbox=${{ matrix.coldboxVersion }} + box install - name: Start ${{ matrix.cfengine }} Server run: | box server start serverConfigFile="server-${{ matrix.cfengine }}.json" --noSaveSettings --debug - # Install Adobe 2021 cfpm modules - if [[ "${{ matrix.cfengine }}" == "adobe@2021" ]] ; then - box run-script install:2021 - fi curl http://127.0.0.1:60299 - name: Run Tests diff --git a/ModuleConfig.cfc b/ModuleConfig.cfc index 3c43589..6fee81b 100644 --- a/ModuleConfig.cfc +++ b/ModuleConfig.cfc @@ -7,12 +7,19 @@ component { // Module Properties - this.title = "Amazon S3 SDK"; - this.author = "Ortus Solutions, Corp"; - this.webURL = "https://www.ortussolutions.com"; - this.description = "This SDK will provide you with Amazon S3 connectivity for any ColdFusion (CFML) application."; - // We will map the models - this.autoMapModels = false; + this.title = "Amazon S3 SDK"; + this.author = "Ortus Solutions, Corp"; + this.webURL = "https://www.ortussolutions.com"; + this.description = "This SDK will provide you with Amazon S3 connectivity for any ColdFusion (CFML) application."; + + // Module Entry Point + this.entryPoint = "s3sdk"; + // Model Namespace + this.modelNamespace = "s3sdk"; + // CF Mapping + this.cfmapping = "s3sdk"; + // Auto-map models + this.autoMapModels = false; /** * Configure @@ -20,27 +27,32 @@ component { function configure(){ // Settings variables.settings = { - accessKey : "", - autoContentType : false, - autoMD5 : false, - awsDomain : "amazonaws.com", - awsRegion : "us-east-1", - debug : false, - defaultACL : "public-read", - defaultBucketName : "", - defaultCacheControl : "no-store, no-cache, must-revalidate", - defaultDelimiter : "/", - defaultStorageClass : "STANDARD", - defaultTimeOut : 300, - encryptionCharset : "utf-8", - retriesOnError : 3, - secretKey : "", - serviceName : "s3", - signatureType : "V4", - ssl : true, - throwOnRequestError : true, - defaultEncryptionAlgorithm : "", - defaultEncryptionKey : "" + accessKey : "", + autoContentType : false, + autoMD5 : false, + awsDomain : "amazonaws.com", + awsRegion : "us-east-1", + debug : false, + defaultACL : "public-read", + defaultBucketName : "", + defaultCacheControl : "no-store, no-cache, must-revalidate", + defaultDelimiter : "/", + defaultStorageClass : "STANDARD", + defaultTimeOut : 300, + encryptionCharset : "utf-8", + retriesOnError : 3, + secretKey : "", + serviceName : "s3", + signatureType : "V4", + ssl : true, + throwOnRequestError : true, + defaultEncryptionAlgorithm : "", + defaultEncryptionKey : "", + defaultObjectOwnership : "ObjectWriter", + defaultBlockPublicAcls : false, + defaultIgnorePublicAcls : false, + defaultBlockPublicPolicy : false, + defaultRestrictPublicBuckets : false }; } @@ -70,7 +82,15 @@ component { .initArg( name = "serviceName", value = variables.settings.serviceName ) .initArg( name = "debug", value = variables.settings.debug ) .initArg( name = "defaultEncryptionAlgorithm", value = variables.settings.defaultEncryptionAlgorithm ) - .initArg( name = "defaultEncryptionKey", value = variables.settings.defaultEncryptionKey ); + .initArg( name = "defaultEncryptionKey", value = variables.settings.defaultEncryptionKey ) + .initArg( name = "defaultObjectOwnership", value = variables.settings.defaultObjectOwnership ) + .initArg( name = "defaultBlockPublicAcls", value = variables.settings.defaultBlockPublicAcls ) + .initArg( name = "defaultIgnorePublicAcls", value = variables.settings.defaultIgnorePublicAcls ) + .initArg( name = "defaultBlockPublicPolicy", value = variables.settings.defaultBlockPublicPolicy ) + .initArg( + name = "defaultRestrictPublicBuckets", + value = variables.settings.defaultRestrictPublicBuckets + ); binder.map( "Sv4Util@s3sdk" ).to( "#moduleMapping#.models.AmazonS3" ); binder.map( "Sv2Util@s3sdk" ).to( "#moduleMapping#.models.AmazonS3" ); diff --git a/box.json b/box.json index 8aa636d..246719b 100644 --- a/box.json +++ b/box.json @@ -1,6 +1,6 @@ { "name":"Amazon S3 SDK", - "version":"5.5.2", + "version":"5.7.1", "slug":"s3sdk", "location":"https://downloads.ortussolutions.com/ortussolutions/coldbox-modules/s3sdk/@build.version@/s3sdk-@build.version@.zip", "author":"Ortus Solutions, Corp", @@ -46,9 +46,6 @@ "format":"cfformat run models,test-harness/tests/**/*.cfc,*.cfc --overwrite", "format:watch":"cfformat watch models,test-harness/tests/**/*.cfc,*.cfc ./.cfformat.json", "format:check":"cfformat check models,test-harness/tests/**/*.cfc,*.cfc", - "cfpm":"echo '\".engine/adobe2021/WEB-INF/cfusion/bin/cfpm.sh\"' | run", - "cfpm:install":"echo '\".engine/adobe2021/WEB-INF/cfusion/bin/cfpm.sh\" install ${1}' | run", - "install:2021":"run-script cfpm:install zip,debugger,mysql", - "install:dependencies":"install && cd test-harness && install" + "install:dependencies":"install --force && cd test-harness && install --force" } } diff --git a/changelog.md b/changelog.md index bfccff4..e6c105a 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ---- +## v5.7.1 => 2023-SEP-21 +### Fixed +* Added `entryPoint`, `modelNamespace` and `cfmapping` keys to ModuleConfig, to ensure mappings for downstream modules are available during framework load +## v5.7.0 => 2023-MAY-03 +### Changed +* Updates permission handling to account for updated AWS default bucket policies +## v5.6.0 => 2023-MAR-07 +### Added +* Support for overriding response headers like content type for pre-signed URLs ## v5.5.2 => 2023-FEB-07 ### Fixed * Multi-part upload concurrency fixes diff --git a/models/AmazonS3.cfc b/models/AmazonS3.cfc index b648d51..f1616d5 100644 --- a/models/AmazonS3.cfc +++ b/models/AmazonS3.cfc @@ -44,6 +44,7 @@ component accessors="true" singleton { property name="defaultBucketName"; property name="defaultCacheControl"; property name="defaultStorageClass"; + property name="defaultObjectOwnership"; property name="defaultACL"; property name="throwOnRequestError"; property name="retriesOnError"; @@ -54,6 +55,11 @@ component accessors="true" singleton { property name="defaultEncryptionAlgorithm"; property name="defaultEncryptionKey"; property name="multiPartByteThreshold"; + property name="defaultBlockPublicAcls"; + property name="defaultIgnorePublicAcls"; + property name="defaultBlockPublicPolicy"; + property name="defaultRestrictPublicBuckets"; + // STATIC Contsants this.ACL_PRIVATE = "private"; @@ -78,51 +84,62 @@ component accessors="true" singleton { /** * Create a new S3SDK Instance * - * @accessKey The Amazon access key. - * @secretKey The Amazon secret key. - * @awsDomain The Domain used S3 Service (amazonws.com, digitalocean.com, storage.googleapis.com). Defaults to amazonws.com - * @awsRegion The Amazon region. Defaults to us-east-1 for amazonaws.com - * @encryptionCharset The charset for the encryption. Defaults to UTF-8. - * @signature The signature version to calculate, "V2" is deprecated but more compatible with other endpoints. "V4" requires Sv4Util.cfc & ESAPI on Lucee. Defaults to V4 - * @ssl True if the request should use SSL. Defaults to true. - * @defaultTimeOut Default HTTP timeout for all requests. Defaults to 300. - * @defaultDelimiter Delimter to use for getBucket calls. "/" is standard to treat keys as file paths - * @defaultBucketName Bucket name to use by default - * @defaultCacheControl Default caching policy for objects. Defaults to: no-store, no-cache, must-revalidate - * @defaultStorageClass Default storage class for objects that affects cost, access speed and durability. Defaults to STANDARD. - * @defaultACL Default access control policy for objects and buckets. Defaults to public-read. - * @throwOnRequestError If an exception should be thrown for request errors. Defaults to true. - * @autoContentType Tries to determine content type of file by file extension. Defaults to false. - * @autoMD5 Calculates MD5 hash of content automatically. Defaults to false. - * @debug Used to turn debugging on or off outside of logbox. Defaults to false. - * @defaultEncryptionAlgorithm The default server side encryption algorithm to use. Usually "AES256". Not needed if using custom defaultEncryptionKey - * @defaultEncryptionKey The default base64 encoded AES 356 bit key for server side encryption. + * @accessKey The Amazon access key. + * @secretKey The Amazon secret key. + * @awsDomain The Domain used S3 Service (amazonws.com, digitalocean.com, storage.googleapis.com). Defaults to amazonws.com + * @awsRegion The Amazon region. Defaults to us-east-1 for amazonaws.com + * @encryptionCharset The charset for the encryption. Defaults to UTF-8. + * @signature The signature version to calculate, "V2" is deprecated but more compatible with other endpoints. "V4" requires Sv4Util.cfc & ESAPI on Lucee. Defaults to V4 + * @ssl True if the request should use SSL. Defaults to true. + * @defaultTimeOut Default HTTP timeout for all requests. Defaults to 300. + * @defaultDelimiter Delimter to use for getBucket calls. "/" is standard to treat keys as file paths + * @defaultBucketName Bucket name to use by default + * @defaultCacheControl Default caching policy for objects. Defaults to: no-store, no-cache, must-revalidate + * @defaultStorageClass Default storage class for objects that affects cost, access speed and durability. Defaults to STANDARD. + * @defaultACL Default access control policy for objects and buckets. Defaults to public-read. + * @throwOnRequestError If an exception should be thrown for request errors. Defaults to true. + * @autoContentType Tries to determine content type of file by file extension. Defaults to false. + * @autoMD5 Calculates MD5 hash of content automatically. Defaults to false. + * @debug Used to turn debugging on or off outside of logbox. Defaults to false. + * @defaultEncryptionAlgorithm The default server side encryption algorithm to use. Usually "AES256". Not needed if using custom defaultEncryptionKey + * @defaultEncryptionKey The default base64 encoded AES 356 bit key for server side encryption. + * @multiPartByteThreshold Min size for multi-part uploads + * @defaultObjectOwnership Default bucket object ownership. One of the values BucketOwnerPreferred, ObjectWriter, BucketOwnerEnforced + * @defaultBlockPublicAcls Specifies whether Amazon S3 should block public access control lists (ACLs) for this bucket and objects in this bucket. + * @defaultIgnorePublicAcls Specifies whether Amazon S3 should block public bucket policies for this bucket. Setting this element to TRUE causes Amazon S3 to reject calls to PUT Bucket policy if the specified bucket policy allows public access. + * @defaultBlockPublicPolicy Specifies whether Amazon S3 should ignore public ACLs for this bucket and objects in this bucket. Setting this element to TRUE causes Amazon S3 to ignore all public ACLs on this bucket and objects in this bucket. + * @defaultRestrictPublicBuckets Specifies whether Amazon S3 should restrict public bucket policies for this bucket. Setting this element to TRUE restricts access to this bucket to only AWS service principals and authorized users within this account if the bucket has a public policy. * * @return An AmazonS3 instance. */ AmazonS3 function init( required string accessKey, required string secretKey, - string awsDomain = "amazonaws.com", - string awsRegion = "", // us-east-1 default for aws - string encryptionCharset = "UTF-8", - string signatureType = "V4", - boolean ssl = true, - string defaultTimeOut = 300, - string defaultDelimiter = "/", - string defaultBucketName = "", - string defaultCacheControl = "no-store, no-cache, must-revalidate", - string defaultStorageClass = this.S3_STANDARD, - string defaultACL = this.ACL_PUBLIC_READ, - string throwOnRequestError = true, - numeric retriesOnError = 3, - boolean autoContentType = false, - boolean autoMD5 = false, - string serviceName = "s3", - boolean debug = false, - string defaultEncryptionAlgorithm = "", - string defaultEncryptionKey = "", - numeric multiPartByteThreshold = 5242880 // 5.2MB is the AWS default minimum size for multipart uploads + string awsDomain = "amazonaws.com", + string awsRegion = "", // us-east-1 default for aws + string encryptionCharset = "UTF-8", + string signatureType = "V4", + boolean ssl = true, + string defaultTimeOut = 300, + string defaultDelimiter = "/", + string defaultBucketName = "", + string defaultCacheControl = "no-store, no-cache, must-revalidate", + string defaultStorageClass = this.S3_STANDARD, + string defaultACL = this.ACL_PUBLIC_READ, + string throwOnRequestError = true, + numeric retriesOnError = 3, + boolean autoContentType = false, + boolean autoMD5 = false, + string serviceName = "s3", + boolean debug = false, + string defaultEncryptionAlgorithm = "", + string defaultEncryptionKey = "", + numeric multiPartByteThreshold = 5242880, // 5.2MB is the AWS default minimum size for multipart uploads + string defaultObjectOwnership = "ObjectWriter", + boolean defaultBlockPublicAcls = false, + boolean defaultIgnorePublicAcls = false, + boolean defaultBlockPublicPolicy = false, + boolean defaultRestrictPublicBuckets = false ){ if ( arguments.awsDomain == "amazonaws.com" && arguments.awsRegion == "" ) { arguments.awsRegion = "us-east-1"; @@ -134,26 +151,31 @@ component accessors="true" singleton { if ( arguments.keyExists( "encryption_charset" ) ) { arguments.encryptionCharset = arguments.encryption_charset; } - variables.accessKey = arguments.accessKey; - variables.secretKey = arguments.secretKey; - variables.encryptionCharset = arguments.encryptionCharset; - variables.signatureType = arguments.signatureType; - variables.awsDomain = arguments.awsDomain; - variables.awsRegion = arguments.awsRegion; - variables.defaultTimeOut = arguments.defaultTimeOut; - variables.defaultDelimiter = arguments.defaultDelimiter; - variables.defaultBucketName = arguments.defaultBucketName; - variables.defaultCacheControl = arguments.defaultCacheControl; - variables.defaultStorageClass = arguments.defaultStorageClass; - variables.defaultACL = arguments.defaultACL; - variables.throwOnRequestError = arguments.throwOnRequestError; - variables.retriesOnError = arguments.retriesOnError; - variables.autoContentType = arguments.autoContentType; - variables.autoMD5 = ( variables.signatureType == "V2" || arguments.autoMD5 ? "auto" : "" ); - variables.serviceName = arguments.serviceName; - variables.defaultEncryptionAlgorithm = arguments.defaultEncryptionAlgorithm; - variables.defaultEncryptionKey = arguments.defaultEncryptionKey; - variables.multiPartByteThreshold = arguments.multiPartByteThreshold; + variables.accessKey = arguments.accessKey; + variables.secretKey = arguments.secretKey; + variables.encryptionCharset = arguments.encryptionCharset; + variables.signatureType = arguments.signatureType; + variables.awsDomain = arguments.awsDomain; + variables.awsRegion = arguments.awsRegion; + variables.defaultTimeOut = arguments.defaultTimeOut; + variables.defaultDelimiter = arguments.defaultDelimiter; + variables.defaultBucketName = arguments.defaultBucketName; + variables.defaultCacheControl = arguments.defaultCacheControl; + variables.defaultStorageClass = arguments.defaultStorageClass; + variables.defaultACL = arguments.defaultACL; + variables.throwOnRequestError = arguments.throwOnRequestError; + variables.retriesOnError = arguments.retriesOnError; + variables.autoContentType = arguments.autoContentType; + variables.autoMD5 = ( variables.signatureType == "V2" || arguments.autoMD5 ? "auto" : "" ); + variables.serviceName = arguments.serviceName; + variables.defaultEncryptionAlgorithm = arguments.defaultEncryptionAlgorithm; + variables.defaultEncryptionKey = arguments.defaultEncryptionKey; + variables.multiPartByteThreshold = arguments.multiPartByteThreshold; + variables.defaultObjectOwnership = arguments.defaultObjectOwnership; + variables.defaultBlockPublicAcls = arguments.defaultBlockPublicAcls; + variables.defaultIgnorePublicAcls = arguments.defaultIgnorePublicAcls; + variables.defaultBlockPublicPolicy = arguments.defaultBlockPublicPolicy; + variables.defaultRestrictPublicBuckets = arguments.defaultRestrictPublicBuckets; // Construct the SSL Domain setSSL( arguments.ssl ); @@ -504,21 +526,41 @@ component accessors="true" singleton { /** * Create a new bucket. * - * @bucketName The name for the new bucket. - * @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL() - * @location The bucket location. + * @bucketName The name for the new bucket. + * @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL() + * @location The bucket location. + * @objectOwnership One of the values BucketOwnerPreferred, ObjectWriter, BucketOwnerEnforced * * @return True if the bucket was created successfully. */ boolean function putBucket( - required string bucketName = variables.defaultBucketName, - string acl = variables.defaultACL, - string location = "USA" + required string bucketName = variables.defaultBucketName, + string acl = variables.defaultACL, + string location = "USA", + string objectOwnership = variables.defaultObjectOwnership, + boolean BlockPublicAcls = false, + boolean IgnorePublicAcls = false, + boolean BlockPublicPolicy = false, + boolean RestrictPublicBuckets = false ){ requireBucketName( arguments.bucketName ); var constraintXML = arguments.location == "EU" ? "EU" : ""; var headers = { "content-type" : "text/xml" }; - applyACLHeaders( headers, arguments.acl ); + + if ( len( arguments.objectOwnership ) ) { + if ( + !listFindNoCase( + "BucketOwnerPreferred,ObjectWriter,BucketOwnerEnforced", + arguments.objectOwnership + ) + ) { + throw( + message = "Invalid value [#arguments.objectOwnership#] for [objectOwnership] when creating bucket.", + detail = "Valid options are: [BucketOwnerPreferred, ObjectWriter, BucketOwnerEnforced]" + ); + } + headers[ "x-amz-object-ownership" ] = arguments.objectOwnership; + } var results = s3Request( method = "PUT", @@ -527,9 +569,106 @@ component accessors="true" singleton { headers = headers ); + // s3 does not provide a way to set this when creating the bucket + putBucketPublicAccess( + arguments.bucketName, + arguments.BlockPublicAcls, + arguments.IgnorePublicAcls, + arguments.BlockPublicPolicy, + arguments.RestrictPublicBuckets + ); + + // Must set ACL in second step in case public access settings above would prevent the ACL from being saved. + putBucketACL( arguments.bucketName, arguments.acl ); + return results.responseheader.status_code == 200; } + /** + * Sets a bucket's ACL. + * + * @bucketName The name for the new bucket. + * @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL() + */ + function putBucketACL( required string bucketName = variables.defaultBucketName, required string acl ){ + requireBucketName( arguments.bucketName ); + + var results = s3Request( + method = "PUT", + resource = arguments.bucketName, + parameters = { "acl" : "" }, + headers = applyACLHeaders( {}, arguments.acl ), + throwOnError = true + ); + } + + /** + * Set the block public access configuration on a bucket + * + * @bucketName The name for the new bucket. + * @BlockPublicAcls Specifies whether Amazon S3 should block public access control lists (ACLs) for this bucket and objects in this bucket. + * @IgnorePublicAcls Specifies whether Amazon S3 should block public bucket policies for this bucket. Setting this element to TRUE causes Amazon S3 to reject calls to PUT Bucket policy if the specified bucket policy allows public access. + * @BlockPublicPolicy Specifies whether Amazon S3 should ignore public ACLs for this bucket and objects in this bucket. Setting this element to TRUE causes Amazon S3 to ignore all public ACLs on this bucket and objects in this bucket. + * @RestrictPublicBuckets Specifies whether Amazon S3 should restrict public bucket policies for this bucket. Setting this element to TRUE restricts access to this bucket to only AWS service principals and authorized users within this account if the bucket has a public policy. + * + * @return True if the bucket was created successfully. + */ + function putBucketPublicAccess( + required string bucketName = variables.defaultBucketName, + boolean BlockPublicAcls = true, + boolean IgnorePublicAcls = true, + boolean BlockPublicPolicy = true, + boolean RestrictPublicBuckets = true + ){ + requireBucketName( arguments.bucketName ); + var body = " + + #uCase( arguments.BlockPublicAcls )# + #uCase( arguments.IgnorePublicAcls )# + #uCase( arguments.BlockPublicPolicy )# + #uCase( arguments.RestrictPublicBuckets )# + "; + + var headers = { "content-type" : "text/xml" }; + + var results = s3Request( + method = "PUT", + resource = arguments.bucketName, + body = body, + headers = headers, + parameters = { "publicAccessBlock" : "" }, + throwOnError = true + ); + return; + } + + /** + * Get the block public access configuration on a bucket + * + * @bucketName The name for the new bucket. + * + * @return struct with keys BlockPublicAcls, IgnorePublicAcls, BlockPublicPolicy, RestrictPublicBuckets + */ + function getBucketPublicAccess( required string bucketName = variables.defaultBucketName ){ + requireBucketName( arguments.bucketName ); + + var results = s3Request( + method = "GET", + resource = arguments.bucketName, + parameters = { "publicAccessBlock" : "" }, + throwOnError = true + ); + var data = xmlParse( results.response ); + return { + "BlockPublicAcls" : data.PublicAccessBlockConfiguration.BlockPublicAcls.XmlText, + "IgnorePublicAcls" : data.PublicAccessBlockConfiguration.IgnorePublicAcls.XmlText, + "BlockPublicPolicy" : data.PublicAccessBlockConfiguration.BlockPublicPolicy.XmlText, + "RestrictPublicBuckets" : data.PublicAccessBlockConfiguration.RestrictPublicBuckets.XmlText + }; + } + + + /** * Checks for the existance of a bucket * @@ -645,13 +784,12 @@ component accessors="true" singleton { arguments.contentType = getFileMimeType( arguments.filepath ); } - var byteCount = getFileInfo( arguments.filepath ).size; + var byteCount = getFileInfo( arguments.filepath ).size; if ( byteCount <= variables.multiPartByteThreshold ) { arguments.data = fileReadBinary( arguments.filepath ); return putObject( argumentCollection = arguments ); } else { - var jFiles = createObject( "java", "java.nio.file.Files" ); var jPath = createObject( "java", "java.nio.file.Paths" ).get( // Java is less lax on slashes than CF, so getCanonicalPath() cleans that up @@ -668,24 +806,24 @@ component accessors="true" singleton { for ( var i = 1; i <= numberOfUploads; i++ ) { var remaining = byteCount - ( ( i - 1 ) * variables.multiPartByteThreshold ); parts.append( { - "uploadId" : uploadId, - "partNumber" : i, - "offset" : (i-1) * variables.multiPartByteThreshold, - "limit" : remaining <= variables.multiPartByteThreshold ? remaining : variables.multiPartByteThreshold, - "timeout" : arguments.HTTPTimeout, - "channel" : jFiles.newByteChannel( jPath, [] ) + "uploadId" : uploadId, + "partNumber" : i, + "offset" : ( i - 1 ) * variables.multiPartByteThreshold, + "limit" : remaining <= variables.multiPartByteThreshold ? remaining : variables.multiPartByteThreshold, + "timeout" : arguments.HTTPTimeout, + "channel" : jFiles.newByteChannel( jPath, [] ) } ); } try { parts = variables.asyncManager.allApply( parts, function( part ){ var channel = part.channel.position( part.offset ); - var buffer = createObject( "java", "java.nio.ByteBuffer" ).allocate( part.limit ); + var buffer = createObject( "java", "java.nio.ByteBuffer" ).allocate( part.limit ); channel.read( buffer ); return { "partNumber" : part.partNumber, "size" : part.limit, - "channel" : part.channel, + "channel" : part.channel, "response" : s3Request( method = "PUT", resource = bucketName & "/" & uri, @@ -695,9 +833,7 @@ component accessors="true" singleton { "uploadId" : part.uploadId, "partNumber" : part.partNumber }, - headers = { - "content-type" : "binary/octet-stream" - } + headers = { "content-type" : "binary/octet-stream" } ) }; } ); @@ -753,10 +889,11 @@ component accessors="true" singleton { arguments.data = fileReadBinary( arguments.filepath ); return putObject( argumentCollection = arguments ); } finally { - parts.each( (p)=>{ - try{ + parts.each( ( p ) => { + try { p.channel.close() - } catch( any e ) {} + } catch ( any e ) { + } } ); } } @@ -1019,15 +1156,16 @@ component accessors="true" singleton { /** * Returns a query string authenticated URL to an object in S3. * - * @bucketName The bucket the object resides in. - * @uri The uri to the object to create a link for. - * @minutesValid The minutes the link is valid for. Defaults to 60 minutes. - * @useSSL Use SSL for the returned url. - * @method HTTP method that will be used - * @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL(). If omitted, any ACL will be allowed when PUTting the file. - * @metaHeaders Additonal metadata headers to add. - * @contentType The object content type for PUT. If omitted, any content-type will be allowed when PUTting the file. - * @encryptionKey The base64 encoded AES 356 bit key for server side encryption. + * @bucketName The bucket the object resides in. + * @uri The uri to the object to create a link for. + * @minutesValid The minutes the link is valid for. Defaults to 60 minutes. + * @useSSL Use SSL for the returned url. + * @method HTTP method that will be used + * @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL(). If omitted, any ACL will be allowed when PUTting the file. + * @metaHeaders Additonal metadata headers to add. + * @contentType The object content type for PUT. If omitted, any content-type will be allowed when PUTting the file. + * @encryptionKey The base64 encoded AES 356 bit key for server side encryption. + * @responseHeaders A struct of headers to be forced for the HTTP response of GET requests. Valid options are content-type, content-language, expires, cache-control, content-disposition, content-encoding * * @return An authenticated url to the resource. */ @@ -1040,7 +1178,8 @@ component accessors="true" singleton { any acl = "", struct metaHeaders = {}, string contentType, - string encryptionKey = variables.defaultEncryptionKey + string encryptionKey = variables.defaultEncryptionKey, + struct responseHeaders = {} ){ requireBucketName( arguments.bucketName ); @@ -1057,13 +1196,35 @@ component accessors="true" singleton { var hostname = "#bucketName#.#variables.URLEndpointHostname#"; + var requestParams = { "X-Amz-Expires" : arguments.minutesValid * 60 }; + var validResponseHeaders = [ + "content-type", + "content-language", + "expires", + "cache-control", + "content-disposition", + "content-encoding" + ]; + responseHeaders.each( ( header, value ) => { + if ( !validResponseHeaders.findNoCase( header ) ) { + throw( + message = "Invalid Reponse Header for signed URL: [#header#].", + detail = "Valid options are: [#validResponseHeaders.toList()#]" + ); + } + if ( header == "content-type" && value == "auto" ) { + value = getFileMimeType( uri ); + } + requestParams[ "response-" & header ] = value; + } ); + var sigData = variables.signatureUtil.generateSignatureData( requestMethod = arguments.method, hostName = hostname, requestURI = arguments.uri, requestBody = "", requestHeaders = headers, - requestParams = { "X-Amz-Expires" : arguments.minutesValid * 60 }, + requestParams = requestParams, accessKey = variables.accessKey, secretKey = variables.secretKey, regionName = variables.awsRegion, @@ -1395,8 +1556,8 @@ component accessors="true" singleton { "message" : "", "responseheader" : {} }; - var param = ""; - var md5 = ""; + var param = ""; + var md5 = ""; // Default Content Type if ( NOT structKeyExists( arguments.headers, "content-type" ) ) { @@ -1428,7 +1589,7 @@ component accessors="true" singleton { // Let the CF engine directly save the file so it can stream large files to disk and not eat up memory cfhttpAttributes[ "file" ] = getFileFromPath( arguments.filename ); cfhttpAttributes[ "path" ] = getDirectoryFromPath( arguments.filename ); - if( !directoryExists( cfhttpAttributes[ "path" ] ) ) { + if ( !directoryExists( cfhttpAttributes[ "path" ] ) ) { directoryCreate( cfhttpAttributes[ "path" ] ); } if ( !isNull( server.lucee ) ) { @@ -1437,7 +1598,6 @@ component accessors="true" singleton { // https://luceeserver.atlassian.net/browse/LDEV-4357 cfhttpAttributes[ "result" ] = ""; } - } cfhttp( @@ -1487,7 +1647,7 @@ component accessors="true" singleton { // Lucee behavior mentioned above regarding file download incompat with Adobe // When Lucee direct-downnloads a file, it doesn't return ANY details from the HTTP request :/ - if( isNull( local.HTTPResults ) || !isStruct( local.HTTPResults ) ) { + if ( isNull( local.HTTPResults ) || !isStruct( local.HTTPResults ) ) { return results; } diff --git a/server-adobe@2021.json b/server-adobe@2021.json index ca0457b..9a47f65 100644 --- a/server-adobe@2021.json +++ b/server-adobe@2021.json @@ -22,5 +22,8 @@ "openBrowser":"false", "cfconfig":{ "file":".cfconfig.json" + }, + "scripts":{ + "onServerInstall":"cfpm install zip,debugger,mysql" } } diff --git a/server-adobe@2023.json b/server-adobe@2023.json new file mode 100644 index 0000000..5347f8e --- /dev/null +++ b/server-adobe@2023.json @@ -0,0 +1,29 @@ +{ + "name":"s3sdk-adobe@2023", + "app":{ + "serverHomeDirectory":".engine/adobe2023", + "cfengine":"adobe@2023.0.0-beta.1" + }, + "web":{ + "http":{ + "port":"60299" + }, + "rewrites":{ + "enable":"true" + }, + "webroot":"test-harness", + "aliases":{ + "/moduleroot/s3sdk":"./" + } + }, + "jvm":{ + "heapSize":"1024" + }, + "openBrowser":"false", + "cfconfig":{ + "file":".cfconfig.json" + }, + "scripts":{ + "onServerInstall":"cfpm install zip,debugger,mysql" + } +} diff --git a/test-harness/box.json b/test-harness/box.json index b5fb75f..992efb8 100644 --- a/test-harness/box.json +++ b/test-harness/box.json @@ -5,7 +5,7 @@ "private":true, "description":"", "dependencies":{ - "coldbox":"^6.0.0" + "coldbox":"^6" }, "devDependencies":{ "testbox":"^4" diff --git a/test-harness/config/Coldbox.cfc b/test-harness/config/Coldbox.cfc index 9e6160d..2de244d 100644 --- a/test-harness/config/Coldbox.cfc +++ b/test-harness/config/Coldbox.cfc @@ -33,6 +33,11 @@ eventCaching = false }; + settings = { + "targetEngine" : getSystemSetting( "ENGINE", "localhost" ), + "coldBoxVersion" : getSystemSetting( "COLDBOX_VERSION", "" ) + }; + // environment settings, create a detectEnvironment() method to detect it yourself. // create a function with the name of the environment so it can be executed if that environment is detected // the value of the environment is a list of regex patterns to match the cgi.http_host. @@ -74,22 +79,24 @@ moduleSettings = { s3sdk = { // Settings - accessKey : getSystemSetting( "AWS_ACCESS_KEY" ), - secretKey : getSystemSetting( "AWS_ACCESS_SECRET" ) + accessKey : getSystemSetting( "AWS_ACCESS_KEY" ), + secretKey : getSystemSetting( "AWS_ACCESS_SECRET" ), + defaultBucketName : getSystemSetting( + "AWS_DEFAULT_BUCKET_NAME", + "ortus3-s3sdk-bdd-#replace( settings.targetEngine, "@", "-" )#-#reReplace( settings.coldBoxVersion, '[^a-zA-Z0-9]', '', 'all' )#" + ), + awsRegion : getSystemSetting( "AWS_REGION" ), + awsDomain : getSystemSetting( "AWS_DOMAIN" ), + ssl : getSystemSetting( "AWS_SSL", true ) } }; } - /** - * Load the Module you are testing - */ - function afterAspectsLoad( event, interceptData, rc, prc ){ - controller.getModuleService() - .registerAndActivateModule( - moduleName = request.MODULE_NAME, - invocationPath = "moduleroot" - ); + function afterAspectsLoad( event, interceptData ){ + controller + .getModuleService() + .registerModule( moduleName = request.MODULE_NAME, invocationPath = "moduleroot" ); } } \ No newline at end of file diff --git a/test-harness/tests/Application.cfc b/test-harness/tests/Application.cfc index eabb7d2..da67592 100644 --- a/test-harness/tests/Application.cfc +++ b/test-harness/tests/Application.cfc @@ -11,7 +11,7 @@ component { request.MODULE_PATH = "s3sdk"; // APPLICATION CFC PROPERTIES - this.name = "#request.MODULE_NAME# Testing Suite"; + this.name = "s3sdk Testing Suite"; this.sessionManagement = true; this.sessionTimeout = createTimespan( 0, 0, 15, 0 ); this.applicationTimeout = createTimespan( 0, 0, 15, 0 ); @@ -26,15 +26,14 @@ component { // The module root path moduleRootPath = reReplaceNoCase( - this.mappings[ "/root" ], - "#request.module_name#(\\|/)test-harness(\\|/)", + rootPath, + "#request.MODULE_PATH#(\\|/)test-harness(\\|/)", "" ); this.mappings[ "/moduleroot" ] = moduleRootPath; this.mappings[ "/#request.MODULE_NAME#" ] = moduleRootPath & "#request.MODULE_NAME#"; - // request start - public boolean function onRequestStart( String targetPage ){ + function onRequestStart( required targetPage ){ // Set a high timeout for long running tests setting requestTimeout ="9999"; // New ColdBox Virtual Application Starter diff --git a/test-harness/tests/specs/AmazonS3Spec.cfc b/test-harness/tests/specs/AmazonS3Spec.cfc index 0ed3925..b99a5d8 100644 --- a/test-harness/tests/specs/AmazonS3Spec.cfc +++ b/test-harness/tests/specs/AmazonS3Spec.cfc @@ -1,38 +1,39 @@ component extends="coldbox.system.testing.BaseTestCase" { - variables.targetEngine = getUtil().getSystemSetting( "ENGINE", "localhost" ); - variables.testBucket = getUtil().getSystemSetting( - "AWS_DEFAULT_BUCKET_NAME", - "ortus2-s3sdk-bdd-#replace( variables.targetEngine, "@", "-" )#" - ); - this.loadColdbox = true; this.unloadColdbox = false; function beforeAll(){ super.beforeAll(); + prepTmpFolder(); + var moduleSettings = getWirebox().getInstance( "box:moduleSettings:s3sdk" ); + + variables.testBucket = moduleSettings.defaultBucketName; + variables.s3 = new s3sdk.models.AmazonS3( - accessKey = getUtil().getSystemSetting( "AWS_ACCESS_KEY" ), - secretKey = getUtil().getSystemSetting( "AWS_ACCESS_SECRET" ), - awsRegion = getUtil().getSystemSetting( "AWS_REGION" ), - awsDomain = getUtil().getSystemSetting( "AWS_DOMAIN" ), - ssl = getUtil().getSystemSetting( "AWS_SSL", true ), - defaultBucketName = variables.testBucket + accessKey = moduleSettings.accessKey, + secretKey = moduleSettings.secretKey, + awsRegion = moduleSettings.awsRegion, + awsDomain = moduleSettings.awsDomain, + ssl = moduleSettings.ssl, + defaultBucketName = moduleSettings.defaultBucketName, + defaultObjectOwnership = moduleSettings.defaultObjectOwnership ); + getWirebox().autowire( s3 ); prepareMock( s3 ); s3.$property( propertyName = "log", mock = createLogStub() ); - try { + //try { s3.putBucket( testBucket ); - } catch ( any e ) { - writeDump( - var = "Error putting test bucket, maybe cached: #e.message# #e.detail#", - output = "console" - ); - } + //} catch ( any e ) { + // writeDump( + // var = "Error putting test bucket, maybe cached: #e.message# #e.detail#", + // output = "console" + // ); + //} } private function prepTmpFolder(){ @@ -130,7 +131,9 @@ component extends="coldbox.system.testing.BaseTestCase" { expandPath( "/tests/tmp/big_file2.txt" ) ); // And confirm a hash of both file contents still matches - expect( hash( fileRead( expandPath( "/tests/tmp/big_file2.txt" ) ) ) ).toBe( hash( fileRead( expandPath( "/tests/tmp/big_file.txt" ) ) ) ) + expect( hash( fileRead( expandPath( "/tests/tmp/big_file2.txt" ) ) ) ).toBe( + hash( fileRead( expandPath( "/tests/tmp/big_file.txt" ) ) ) + ) } ); it( @@ -462,6 +465,41 @@ component extends="coldbox.system.testing.BaseTestCase" { expect( results ).toBeTrue(); s3.putBucket( testBucket ); } ); + + it( "can get bucketPublicAccess", function(){ + var results = s3.getBucketPublicAccess( testBucket ); + expect( results ).toHaveKey( "BlockPublicAcls" ); + expect( results ).toHaveKey( "IgnorePublicAcls" ); + expect( results ).toHaveKey( "BlockPublicPolicy" ); + expect( results ).toHaveKey( "RestrictPublicBuckets" ); + + expect( results.BlockPublicAcls ).toBeBoolean(); + expect( results.IgnorePublicAcls ).toBeBoolean(); + expect( results.BlockPublicPolicy ).toBeBoolean(); + expect( results.RestrictPublicBuckets ).toBeBoolean(); + } ); + + it( "can set bucketPublicAccess", function(){ + s3.putBucketPublicAccess( testBucket, true, true, true, true ); + var results = s3.getBucketPublicAccess( testBucket ); + + expect( results.BlockPublicAcls ).toBeTrue(); + expect( results.IgnorePublicAcls ).toBeTrue(); + expect( results.BlockPublicPolicy ).toBeTrue(); + expect( results.RestrictPublicBuckets ).toBeTrue(); + + s3.putBucketPublicAccess( testBucket, false, false, false, false ); + var results = s3.getBucketPublicAccess( testBucket ); + + expect( results.BlockPublicAcls ).toBeFalse(); + expect( results.IgnorePublicAcls ).toBeFalse(); + expect( results.BlockPublicPolicy ).toBeFalse(); + expect( results.RestrictPublicBuckets ).toBeFalse(); + } ); + + it( "can set bucket ACL", function(){ + s3.putBucketACL( testBucket, "private" ); + } ); } ); describe( "Presigned URL", function(){ @@ -600,6 +638,61 @@ component extends="coldbox.system.testing.BaseTestCase" { }; expect( local.cfhttp.Responseheader.status_code ?: 0 ).toBe( "403", local.cfhttp.fileContent ); } ); + + it( "Can use presigned URL with forced response headers", function(){ + s3.putObject( testBucket, "example.txt", "Hello, world!" ); + var presignedURL = s3.getAuthenticatedURL( + bucketName = testBucket, + uri = "example.txt", + responseHeaders = { + "content-type" : "custom-type", + "content-language" : "custom-language", + "expires" : "custom-expires", + "cache-control" : "custom-cache", + "content-disposition" : "custom-disposition", + "content-encoding" : "custom-encoding" + } + ); + cfhttp( url = "#presignedURL#", result = "local.cfhttp" ); + + expect( local.cfhttp.Responseheader.status_code ?: 0 ).toBe( "200", local.cfhttp.fileContent ); + expect( local.cfhttp.fileContent ).toBe( "Hello, world!" ); + expect( local.cfhttp.Responseheader[ "content-type" ] ).toBe( "custom-type" ); + expect( local.cfhttp.Responseheader[ "content-language" ] ).toBe( "custom-language" ); + expect( local.cfhttp.Responseheader[ "expires" ] ).toBe( "custom-expires" ); + expect( local.cfhttp.Responseheader[ "cache-control" ] ).toBe( "custom-cache" ); + expect( local.cfhttp.Responseheader[ "content-disposition" ] ).toBe( "custom-disposition" ); + expect( local.cfhttp.Responseheader[ "content-encoding" ] ).toBe( "custom-encoding" ); + } ); + + it( "Can use presigned URL with auto response content type", function(){ + s3.putObject( + testBucket, + "example.txt", + "Hello, world!", + "", + "wacky-content-type" + ); + var presignedURL = s3.getAuthenticatedURL( + bucketName = testBucket, + uri = "example.txt", + responseHeaders = { "content-type" : "auto" } + ); + cfhttp( url = "#presignedURL#", result = "local.cfhttp" ); + + expect( local.cfhttp.Responseheader.status_code ?: 0 ).toBe( "200", local.cfhttp.fileContent ); + expect( local.cfhttp.fileContent ).toBe( "Hello, world!" ); + // Our explicit content type when storing the file is ignored and the corret type is automatically returned based on MIME type + expect( local.cfhttp.Responseheader[ "content-type" ] ).toBe( "text/plain" ); + } ); + + it( "Creating presigned URL with invalid response header throws error", function(){ + expect( () => s3.getAuthenticatedURL( + bucketName = testBucket, + uri = "example.txt", + responseHeaders = { "fake" : "" } + ) ).toThrow(); + } ); } ); } ); diff --git a/test-harness/tests/specs/Sv4UtilSpec.cfc b/test-harness/tests/specs/Sv4UtilSpec.cfc index 7c09237..ac90fbf 100644 --- a/test-harness/tests/specs/Sv4UtilSpec.cfc +++ b/test-harness/tests/specs/Sv4UtilSpec.cfc @@ -1,8 +1,5 @@ component extends="coldbox.system.testing.BaseTestCase" { - variables.targetEngine = getUtil().getSystemSetting( "ENGINE", "localhost" ); - variables.testBucket = "ortus-s3sdk-bdd-#replace( variables.targetEngine, "@", "-" )#"; - function beforeAll(){ variables.sv4 = new s3sdk.models.Sv4Util();