From 37933ed29641506f72a62081d4b9b9a2eeac74ba Mon Sep 17 00:00:00 2001 From: Piotr Srebniak Date: Fri, 25 Oct 2024 13:18:09 +0200 Subject: [PATCH] fix(374): fixes #374, add unit tests (#378) Co-authored-by: Piotr Srebniak --- .gitignore | 8 + .npmignore | 1 + .projen/tasks.json | 69 ++ package.json | 5 + src/s3/destination.ts | 2 +- .../TestStack.assets.json | 71 ++ .../TestStack.template.json | 756 ++++++++++++++++++ test/integ/ondemand-s3-to-s3.integ.ts | 70 ++ test/s3/destination.test.ts | 200 +++++ test/s3/source.test.ts | 106 +++ 10 files changed, 1287 insertions(+), 1 deletion(-) create mode 100644 test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.assets.json create mode 100644 test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.template.json create mode 100644 test/integ/ondemand-s3-to-s3.integ.ts create mode 100644 test/s3/destination.test.ts create mode 100644 test/s3/source.test.ts diff --git a/.gitignore b/.gitignore index 7a369fad..861040e7 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,14 @@ test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/manifest.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/manifest.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/tree.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/tree.json +test/integ/ondemand-s3-to-s3.integ.snapshot/asset.* +test/integ/ondemand-s3-to-s3.integ.snapshot/**/asset.* +test/integ/ondemand-s3-to-s3.integ.snapshot/cdk.out +test/integ/ondemand-s3-to-s3.integ.snapshot/**/cdk.out +test/integ/ondemand-s3-to-s3.integ.snapshot/manifest.json +test/integ/ondemand-s3-to-s3.integ.snapshot/**/manifest.json +test/integ/ondemand-s3-to-s3.integ.snapshot/tree.json +test/integ/ondemand-s3-to-s3.integ.snapshot/**/tree.json test/integ/ondemand-s3-to-snowflake.integ.snapshot/asset.* test/integ/ondemand-s3-to-snowflake.integ.snapshot/**/asset.* test/integ/ondemand-s3-to-snowflake.integ.snapshot/cdk.out diff --git a/.npmignore b/.npmignore index 076283bb..b43696b4 100644 --- a/.npmignore +++ b/.npmignore @@ -30,6 +30,7 @@ test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot test/integ/ondemand-mailchimp-to-s3.integ.snapshot test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot +test/integ/ondemand-s3-to-s3.integ.snapshot test/integ/ondemand-s3-to-snowflake.integ.snapshot test/integ/ondemand-salesforce-to-redshift.integ.snapshot test/integ/ondemand-salesforce-to-s3.integ.snapshot diff --git a/.projen/tasks.json b/.projen/tasks.json index e2dca7df..5b46d6d6 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -629,6 +629,69 @@ } ] }, + "integ:ondemand-s3-to-s3:assert": { + "name": "integ:ondemand-s3-to-s3:assert", + "description": "assert the snapshot of integration test 'ondemand-s3-to-s3'", + "steps": [ + { + "exec": "[ -d \"test/integ/ondemand-s3-to-s3.integ.snapshot\" ] || (echo \"No snapshot available for integration test 'ondemand-s3-to-s3'. Run 'projen integ:ondemand-s3-to-s3:deploy' to capture.\" && exit 1)" + }, + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-s3-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/integ/.tmp/ondemand-s3-to-s3.integ/assert.cdk.out > /dev/null" + }, + { + "exec": "diff -r -x asset.* -x cdk.out -x manifest.json -x tree.json test/integ/ondemand-s3-to-s3.integ.snapshot/ test/integ/.tmp/ondemand-s3-to-s3.integ/assert.cdk.out/" + } + ] + }, + "integ:ondemand-s3-to-s3:deploy": { + "name": "integ:ondemand-s3-to-s3:deploy", + "description": "deploy integration test 'ondemand-s3-to-s3' and capture snapshot", + "steps": [ + { + "exec": "rm -fr test/integ/.tmp/ondemand-s3-to-s3.integ/deploy.cdk.out" + }, + { + "exec": "cdk deploy --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-s3-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' --require-approval=never -o test/integ/.tmp/ondemand-s3-to-s3.integ/deploy.cdk.out" + }, + { + "exec": "rm -fr test/integ/ondemand-s3-to-s3.integ.snapshot" + }, + { + "exec": "mv test/integ/.tmp/ondemand-s3-to-s3.integ/deploy.cdk.out test/integ/ondemand-s3-to-s3.integ.snapshot" + }, + { + "spawn": "integ:ondemand-s3-to-s3:destroy" + } + ] + }, + "integ:ondemand-s3-to-s3:destroy": { + "name": "integ:ondemand-s3-to-s3:destroy", + "description": "destroy integration test 'ondemand-s3-to-s3'", + "steps": [ + { + "exec": "cdk destroy --app test/integ/ondemand-s3-to-s3.integ.snapshot '**' --no-version-reporting" + } + ] + }, + "integ:ondemand-s3-to-s3:snapshot": { + "name": "integ:ondemand-s3-to-s3:snapshot", + "description": "update snapshot for integration test \"ondemand-s3-to-s3\"", + "steps": [ + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-s3-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/integ/ondemand-s3-to-s3.integ.snapshot > /dev/null" + } + ] + }, + "integ:ondemand-s3-to-s3:watch": { + "name": "integ:ondemand-s3-to-s3:watch", + "description": "watch integration test 'ondemand-s3-to-s3' (without updating snapshots)", + "steps": [ + { + "exec": "cdk watch --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-s3-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' -o test/integ/.tmp/ondemand-s3-to-s3.integ/deploy.cdk.out" + } + ] + }, "integ:ondemand-s3-to-snowflake:assert": { "name": "integ:ondemand-s3-to-snowflake:assert", "description": "assert the snapshot of integration test 'ondemand-s3-to-snowflake'", @@ -1158,6 +1221,9 @@ { "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:snapshot" }, + { + "spawn": "integ:ondemand-s3-to-s3:snapshot" + }, { "spawn": "integ:ondemand-s3-to-snowflake:snapshot" }, @@ -1364,6 +1430,9 @@ { "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:assert" }, + { + "spawn": "integ:ondemand-s3-to-s3:assert" + }, { "spawn": "integ:ondemand-s3-to-snowflake:assert" }, diff --git a/package.json b/package.json index bee62fc1..ce54a8c5 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,11 @@ "integ:ondemand-microsoftsharepointonline-to-s3:destroy": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:destroy", "integ:ondemand-microsoftsharepointonline-to-s3:snapshot": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:snapshot", "integ:ondemand-microsoftsharepointonline-to-s3:watch": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:watch", + "integ:ondemand-s3-to-s3:assert": "npx projen integ:ondemand-s3-to-s3:assert", + "integ:ondemand-s3-to-s3:deploy": "npx projen integ:ondemand-s3-to-s3:deploy", + "integ:ondemand-s3-to-s3:destroy": "npx projen integ:ondemand-s3-to-s3:destroy", + "integ:ondemand-s3-to-s3:snapshot": "npx projen integ:ondemand-s3-to-s3:snapshot", + "integ:ondemand-s3-to-s3:watch": "npx projen integ:ondemand-s3-to-s3:watch", "integ:ondemand-s3-to-snowflake:assert": "npx projen integ:ondemand-s3-to-snowflake:assert", "integ:ondemand-s3-to-snowflake:deploy": "npx projen integ:ondemand-s3-to-snowflake:deploy", "integ:ondemand-s3-to-snowflake:destroy": "npx projen integ:ondemand-s3-to-snowflake:destroy", diff --git a/src/s3/destination.ts b/src/s3/destination.ts index c8e344b7..9f089e85 100644 --- a/src/s3/destination.ts +++ b/src/s3/destination.ts @@ -76,7 +76,7 @@ export enum S3OutputFileType { /** * Parquet file type */ - PARQUET = 'Parquet', + PARQUET = 'PARQUET', } export enum S3OutputFilePrefixHierarchy { diff --git a/test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.assets.json new file mode 100644 index 00000000..e7ca5eca --- /dev/null +++ b/test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.assets.json @@ -0,0 +1,71 @@ +{ + "version": "36.3.0", + "files": { + "faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6": { + "source": { + "path": "asset.faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "3322b7049fb0ed2b7cbb644a2ada8d1116ff80c32dca89e6ada846b5de26f961": { + "source": { + "path": "asset.3322b7049fb0ed2b7cbb644a2ada8d1116ff80c32dca89e6ada846b5de26f961.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "3322b7049fb0ed2b7cbb644a2ada8d1116ff80c32dca89e6ada846b5de26f961.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "0158f40002a8c211635388a87874fd4dcc3d68f525fe08a0fe0f014069ae539c": { + "source": { + "path": "asset.0158f40002a8c211635388a87874fd4dcc3d68f525fe08a0fe0f014069ae539c", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "0158f40002a8c211635388a87874fd4dcc3d68f525fe08a0fe0f014069ae539c.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "b7a00a65c9fbbf4751a2ff83a7978df76a9502f91f38166662688fee371134ce": { + "source": { + "path": "asset.b7a00a65c9fbbf4751a2ff83a7978df76a9502f91f38166662688fee371134ce", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "b7a00a65c9fbbf4751a2ff83a7978df76a9502f91f38166662688fee371134ce.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "04bac35225200905aca61924fe664f2403663a62d823cdc3d4aec2dc54aaf0fe": { + "source": { + "path": "TestStack.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "04bac35225200905aca61924fe664f2403663a62d823cdc3d4aec2dc54aaf0fe.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.template.json b/test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.template.json new file mode 100644 index 00000000..871139cd --- /dev/null +++ b/test/integ/ondemand-s3-to-s3.integ.snapshot/TestStack.template.json @@ -0,0 +1,756 @@ +{ + "Resources": { + "TestSourceBucketC9809AD6": { + "Type": "AWS::S3::Bucket", + "Properties": { + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + }, + { + "Key": "aws-cdk:cr-owned:account:a91b1dfe", + "Value": "true" + } + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "TestSourceBucketPolicy56288476": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "TestSourceBucketC9809AD6" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:PutBucketPolicy", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestSourceBucketC9809AD6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestSourceBucketC9809AD6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Condition": { + "StringEquals": { + "aws:SourceAccount": { + "Ref": "AWS::AccountId" + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "appflow.amazonaws.com" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestSourceBucketC9809AD6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestSourceBucketC9809AD6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "TestSourceBucketAutoDeleteObjectsCustomResource6CB80E00": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestSourceBucketC9809AD6" + } + }, + "DependsOn": [ + "TestSourceBucketPolicy56288476" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6.zip" + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region" + }, + "value" + ] + }, + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "TestSourceBucketC9809AD6" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "TestDestinationBucketAF81892F": { + "Type": "AWS::S3::Bucket", + "Properties": { + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "TestDestinationBucketPolicyF3C5CBC8": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "TestDestinationBucketAF81892F" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:PutBucketPolicy", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestDestinationBucketAF81892F", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestDestinationBucketAF81892F", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucketMultipartUploads", + "s3:GetBucketAcl", + "s3:PutObjectAcl" + ], + "Condition": { + "StringEquals": { + "aws:SourceAccount": { + "Ref": "AWS::AccountId" + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "appflow.amazonaws.com" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestDestinationBucketAF81892F", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestDestinationBucketAF81892F", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "TestDestinationBucketAutoDeleteObjectsCustomResource7768BEBD": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestDestinationBucketAF81892F" + } + }, + "DependsOn": [ + "TestDestinationBucketPolicyF3C5CBC8" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "TestDeploymentAwsCliLayerACB69B63": { + "Type": "AWS::Lambda::LayerVersion", + "Properties": { + "Content": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "3322b7049fb0ed2b7cbb644a2ada8d1116ff80c32dca89e6ada846b5de26f961.zip" + }, + "Description": "/opt/awscli/aws" + } + }, + "TestDeploymentCustomResource6C695E22": { + "Type": "Custom::CDKBucketDeployment", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536", + "Arn" + ] + }, + "SourceBucketNames": [ + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ], + "SourceObjectKeys": [ + "b7a00a65c9fbbf4751a2ff83a7978df76a9502f91f38166662688fee371134ce.zip" + ], + "SourceMarkers": [ + {} + ], + "DestinationBucketName": { + "Ref": "TestSourceBucketC9809AD6" + }, + "DestinationBucketKeyPrefix": "account", + "Prune": true + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRoleDefaultPolicy88902FDF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "TestSourceBucketC9809AD6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestSourceBucketC9809AD6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRoleDefaultPolicy88902FDF", + "Roles": [ + { + "Ref": "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265" + } + ] + } + }, + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "0158f40002a8c211635388a87874fd4dcc3d68f525fe08a0fe0f014069ae539c.zip" + }, + "Environment": { + "Variables": { + "AWS_CA_BUNDLE": "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" + } + }, + "Handler": "index.handler", + "Layers": [ + { + "Ref": "TestDeploymentAwsCliLayerACB69B63" + } + ], + "Role": { + "Fn::GetAtt": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265", + "Arn" + ] + }, + "Runtime": "python3.11", + "Timeout": 900 + }, + "DependsOn": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRoleDefaultPolicy88902FDF", + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265" + ] + }, + "OnDemandFlow4ECA54C5": { + "Type": "AWS::AppFlow::Flow", + "Properties": { + "DestinationFlowConfigList": [ + { + "ConnectorType": "S3", + "DestinationConnectorProperties": { + "S3": { + "BucketName": { + "Ref": "TestDestinationBucketAF81892F" + }, + "S3OutputFormatConfig": { + "AggregationConfig": { + "AggregationType": "None" + }, + "FileType": "PARQUET", + "PrefixConfig": { + "PathPrefixHierarchy": [ + "SCHEMA_VERSION", + "EXECUTION_ID" + ], + "PrefixType": "FILENAME" + }, + "PreserveSourceDataTyping": true + } + } + } + } + ], + "FlowName": "TestStackOnDemandFlow53F97EB9", + "SourceFlowConfig": { + "ConnectorType": "S3", + "SourceConnectorProperties": { + "S3": { + "BucketName": { + "Ref": "TestSourceBucketC9809AD6" + }, + "BucketPrefix": "account", + "S3InputFormatConfig": { + "S3InputFileType": "JSON" + } + } + } + }, + "Tasks": [ + { + "ConnectorOperator": { + "S3": "NO_OP" + }, + "SourceFields": [], + "TaskProperties": [ + { + "Key": "EXCLUDE_SOURCE_FIELDS_LIST", + "Value": "[]" + } + ], + "TaskType": "Map_all" + } + ], + "TriggerConfig": { + "TriggerType": "OnDemand" + } + }, + "DependsOn": [ + "TestDeploymentAwsCliLayerACB69B63", + "TestDeploymentCustomResource6C695E22", + "TestDestinationBucketAutoDeleteObjectsCustomResource7768BEBD", + "TestDestinationBucketPolicyF3C5CBC8", + "TestDestinationBucketAF81892F", + "TestSourceBucketAutoDeleteObjectsCustomResource6CB80E00", + "TestSourceBucketPolicy56288476", + "TestSourceBucketC9809AD6" + ] + } + }, + "Mappings": { + "LatestNodeRuntimeMap": { + "af-south-1": { + "value": "nodejs20.x" + }, + "ap-east-1": { + "value": "nodejs20.x" + }, + "ap-northeast-1": { + "value": "nodejs20.x" + }, + "ap-northeast-2": { + "value": "nodejs20.x" + }, + "ap-northeast-3": { + "value": "nodejs20.x" + }, + "ap-south-1": { + "value": "nodejs20.x" + }, + "ap-south-2": { + "value": "nodejs20.x" + }, + "ap-southeast-1": { + "value": "nodejs20.x" + }, + "ap-southeast-2": { + "value": "nodejs20.x" + }, + "ap-southeast-3": { + "value": "nodejs20.x" + }, + "ap-southeast-4": { + "value": "nodejs20.x" + }, + "ap-southeast-5": { + "value": "nodejs20.x" + }, + "ap-southeast-7": { + "value": "nodejs20.x" + }, + "ca-central-1": { + "value": "nodejs20.x" + }, + "ca-west-1": { + "value": "nodejs20.x" + }, + "cn-north-1": { + "value": "nodejs18.x" + }, + "cn-northwest-1": { + "value": "nodejs18.x" + }, + "eu-central-1": { + "value": "nodejs20.x" + }, + "eu-central-2": { + "value": "nodejs20.x" + }, + "eu-isoe-west-1": { + "value": "nodejs18.x" + }, + "eu-north-1": { + "value": "nodejs20.x" + }, + "eu-south-1": { + "value": "nodejs20.x" + }, + "eu-south-2": { + "value": "nodejs20.x" + }, + "eu-west-1": { + "value": "nodejs20.x" + }, + "eu-west-2": { + "value": "nodejs20.x" + }, + "eu-west-3": { + "value": "nodejs20.x" + }, + "il-central-1": { + "value": "nodejs20.x" + }, + "me-central-1": { + "value": "nodejs20.x" + }, + "me-south-1": { + "value": "nodejs20.x" + }, + "mx-central-1": { + "value": "nodejs20.x" + }, + "sa-east-1": { + "value": "nodejs20.x" + }, + "us-east-1": { + "value": "nodejs20.x" + }, + "us-east-2": { + "value": "nodejs20.x" + }, + "us-gov-east-1": { + "value": "nodejs18.x" + }, + "us-gov-west-1": { + "value": "nodejs18.x" + }, + "us-iso-east-1": { + "value": "nodejs18.x" + }, + "us-iso-west-1": { + "value": "nodejs18.x" + }, + "us-isob-east-1": { + "value": "nodejs18.x" + }, + "us-west-1": { + "value": "nodejs20.x" + }, + "us-west-2": { + "value": "nodejs20.x" + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/test/integ/ondemand-s3-to-s3.integ.ts b/test/integ/ondemand-s3-to-s3.integ.ts new file mode 100644 index 00000000..8660bbcb --- /dev/null +++ b/test/integ/ondemand-s3-to-s3.integ.ts @@ -0,0 +1,70 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { App, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { BucketDeployment, Source } from 'aws-cdk-lib/aws-s3-deployment'; +import { + Mapping, + OnDemandFlow, + S3Destination, S3InputFileType, S3OutputAggregationType, + S3OutputFilePrefixHierarchy, S3OutputFilePrefixType, + S3OutputFileType, S3Source, +} from '../../src'; + +const app = new App({ + treeMetadata: false, +}); + +const stack = new Stack(app, 'TestStack'); + +const sourceBucket = new Bucket(stack, 'TestSourceBucket', { + autoDeleteObjects: true, + removalPolicy: RemovalPolicy.DESTROY, +}); + +const destinationBucket = new Bucket(stack, 'TestDestinationBucket', { + autoDeleteObjects: true, + removalPolicy: RemovalPolicy.DESTROY, +}); + +const deployment = new BucketDeployment(stack, 'TestDeployment', { + sources: [Source.jsonData('init.json', { Name: 'name' })], + destinationBucket: sourceBucket, + destinationKeyPrefix: 'account', +}); + +const source = new S3Source({ + bucket: sourceBucket, + prefix: 'account', + format: { + type: S3InputFileType.JSON, + }, +}); + +const destination = new S3Destination({ + location: { bucket: destinationBucket }, + formatting: { + fileType: S3OutputFileType.PARQUET, + filePrefix: { + type: S3OutputFilePrefixType.FILENAME, + hierarchy: [ + S3OutputFilePrefixHierarchy.SCHEMA_VERSION, + S3OutputFilePrefixHierarchy.EXECUTION_ID, + ], + }, + aggregation: { type: S3OutputAggregationType.NONE }, + preserveSourceDataTypes: true, + }, +}); + +const flow = new OnDemandFlow(stack, 'OnDemandFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], +}); + +flow.node.addDependency(deployment); + +app.synth(); \ No newline at end of file diff --git a/test/s3/destination.test.ts b/test/s3/destination.test.ts new file mode 100644 index 00000000..1da6d7ac --- /dev/null +++ b/test/s3/destination.test.ts @@ -0,0 +1,200 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { Database } from '@aws-cdk/aws-glue-alpha'; +import { Stack } from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { + OnDemandFlow, + S3Destination, + Mapping, + S3Source, + S3ConnectorType, + S3OutputAggregationType, + S3OutputFileType, + S3OutputFilePrefixHierarchy, + S3OutputFilePrefixFormat, S3OutputFilePrefixType, +} from '../../src'; + +describe('S3Destination', () => { + + test('Destination', () => { + const stack = new Stack(undefined, 'TestStack'); + + const bucket = new Bucket(stack, 'TestBucket', { + bucketName: 'test-bucket', + }); + + const source = new S3Source({ + bucket: bucket, + prefix: 'prefix', + }); + + const destination = new S3Destination({ + location: { + bucket, + }, + formatting: { + fileType: S3OutputFileType.PARQUET, + filePrefix: { + type: S3OutputFilePrefixType.FILENAME, + hierarchy: [ + S3OutputFilePrefixHierarchy.SCHEMA_VERSION, + S3OutputFilePrefixHierarchy.EXECUTION_ID, + ], + }, + aggregation: { type: S3OutputAggregationType.NONE }, + preserveSourceDataTypes: true, + }, + catalog: { + database: new Database(stack, 'TestDatabase', { databaseName: 'testdb' }), + tablePrefix: 'db_prefix', + }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + const expectedConnectorType = S3ConnectorType.instance; + expect(destination.connectorType.asProfileConnectorLabel).toEqual(expectedConnectorType.asProfileConnectorLabel); + expect(destination.connectorType.asProfileConnectorType).toEqual(expectedConnectorType.asProfileConnectorType); + expect(destination.connectorType.asTaskConnectorOperatorOrigin).toEqual(expectedConnectorType.asTaskConnectorOperatorOrigin); + expect(destination.connectorType.isCustom).toEqual(expectedConnectorType.isCustom); + }); + + test('Flow destination template is valid', () => { + const stack = new Stack(undefined, 'TestStack'); + + const bucket = new Bucket(stack, 'TestBucket', { + bucketName: 'test-bucket', + }); + + const source = new S3Source({ + bucket: bucket, + prefix: 'prefix', + }); + + const destination = new S3Destination({ + location: { + bucket, + }, + formatting: { + fileType: S3OutputFileType.PARQUET, + filePrefix: { + type: S3OutputFilePrefixType.FILENAME, + hierarchy: [ + S3OutputFilePrefixHierarchy.SCHEMA_VERSION, + S3OutputFilePrefixHierarchy.EXECUTION_ID, + ], + }, + aggregation: { type: S3OutputAggregationType.NONE }, + preserveSourceDataTypes: true, + }, + catalog: { + database: new Database(stack, 'TestDatabase', { databaseName: 'testdb' }), + tablePrefix: 'db_prefix', + }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + const template = Template.fromStack(stack); + template.hasResourceProperties('AWS::AppFlow::Flow', { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: { + Ref: 'TestBucket560B80BC', + }, + S3OutputFormatConfig: { + AggregationConfig: { + AggregationType: 'None', + }, + FileType: 'PARQUET', + PrefixConfig: { + PathPrefixHierarchy: [ + 'SCHEMA_VERSION', + 'EXECUTION_ID', + ], + PrefixType: 'FILENAME', + }, + PreserveSourceDataTyping: true, + }, + }, + }, + }, + ], + MetadataCatalogConfig: { + GlueDataCatalog: { + TablePrefix: 'db_prefix', + }, + }, + }); + }); + + test('Only known S3OutputAggregationType specified', () => { + const expectedTypes = [ + 'None', + 'SingleFile', + ]; + const actualTypes = Object.values(S3OutputAggregationType); + + expect(actualTypes).toEqual(expectedTypes); + }); + + test('Only known S3OutputFileType specified', () => { + const expectedTypes = [ + 'CSV', + 'JSON', + 'PARQUET', + ]; + const actualTypes = Object.values(S3OutputFileType); + + expect(actualTypes).toEqual(expectedTypes); + }); + + test('Only known S3OutputFilePrefixHierarchy specified', () => { + const expectedTypes = [ + 'EXECUTION_ID', + 'SCHEMA_VERSION', + ]; + const actualTypes = Object.values(S3OutputFilePrefixHierarchy); + + expect(actualTypes).toEqual(expectedTypes); + }); + + test('Only known S3OutputFilePrefixFormat specified', () => { + const expectedTypes = [ + 'DAY', + 'HOUR', + 'MINUTE', + 'MONTH', + 'YEAR', + ]; + const actualTypes = Object.values(S3OutputFilePrefixFormat); + + expect(actualTypes).toEqual(expectedTypes); + }); + + test('Only known S3OutputFilePrefixType specified', () => { + const expectedTypes = [ + 'FILENAME', + 'PATH', + 'PATH_AND_FILE', + ]; + const actualTypes = Object.values(S3OutputFilePrefixType); + + expect(actualTypes).toEqual(expectedTypes); + }); +}); diff --git a/test/s3/source.test.ts b/test/s3/source.test.ts new file mode 100644 index 00000000..aa1ac185 --- /dev/null +++ b/test/s3/source.test.ts @@ -0,0 +1,106 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { Stack } from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { + OnDemandFlow, + S3Destination, + Mapping, S3Source, S3InputFileType, S3ConnectorType, +} from '../../src'; + +describe('S3Source', () => { + + test('Source', () => { + const stack = new Stack(undefined, 'TestStack'); + + const bucket = new Bucket(stack, 'TestBucket', { + bucketName: 'test-bucket', + }); + + const source = new S3Source({ + bucket: bucket, + prefix: 'prefix', + format: { + type: S3InputFileType.CSV, + }, + }); + + const destination = new S3Destination({ + location: { + bucket, + }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + + const expectedConnectorType = S3ConnectorType.instance; + expect(source.connectorType.asProfileConnectorLabel).toEqual(expectedConnectorType.asProfileConnectorLabel); + expect(source.connectorType.asProfileConnectorType).toEqual(expectedConnectorType.asProfileConnectorType); + expect(source.connectorType.asTaskConnectorOperatorOrigin).toEqual(expectedConnectorType.asTaskConnectorOperatorOrigin); + expect(source.connectorType.isCustom).toEqual(expectedConnectorType.isCustom); + }); + + test('Flow source template is valid', () => { + const stack = new Stack(undefined, 'TestStack'); + + const bucket = new Bucket(stack, 'TestBucket', { + bucketName: 'test-bucket', + }); + + const source = new S3Source({ + bucket: bucket, + prefix: 'prefix', + format: { + type: S3InputFileType.CSV, + }, + }); + + const destination = new S3Destination({ + location: { + bucket, + }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + const template = Template.fromStack(stack); + template.hasResourceProperties('AWS::AppFlow::Flow', { + SourceFlowConfig: { + ConnectorType: 'S3', + SourceConnectorProperties: { + S3: { + BucketName: { + Ref: 'TestBucket560B80BC', + }, + BucketPrefix: 'prefix', + S3InputFormatConfig: { + S3InputFileType: 'CSV', + }, + }, + }, + }, + }); + }); + + test('Only known S3InputFileType specified', () => { + const expectedTypes = [ + 'CSV', + 'JSON', + ]; + const actualTypes = Object.values(S3InputFileType); + + expect(actualTypes).toEqual(expectedTypes); + }); +});