Merge pull request #134 from CDLUC3/dynamo-index

Dynamo index
CDLUC3 · May 7, 2024 · 77b6602 · 77b6602
2 parents e68d635 + 01378b1
commit 77b6602
Show file tree

Hide file tree

Showing 33 changed files with 801 additions and 553 deletions.
diff --git a/config/dev/regional/dynamo-index-table.yaml b/config/dev/regional/dynamo-index-table.yaml
@@ -0,0 +1,19 @@
+template:
+  path: dynamo-index-table.yaml
+  type: file
+
+parameters:
+  Env: !stack_attr sceptre_user_data.env
+
+  Domain: !stack_attr sceptre_user_data.domain
+
+  SsmPath: !stack_attr sceptre_user_data.ssm_path
+
+  # Dynamo settings
+  #   See: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html
+  DynamoTableClass: 'STANDARD'
+  DynamoEnableContributorInsights: 'true'
+  DynamoEnablePointInTimeRecovery: 'true'
+  DynamoBillingMode: 'PAY_PER_REQUEST'
+  DynamoReadCapacityUnits: '8'
+  DynamoWriteCapacityUnits: '30'
diff --git a/config/dev/regional/lambda-iam.yaml b/config/dev/regional/lambda-iam.yaml
@@ -5,8 +5,13 @@ template:
 parameters:
   Env: !stack_attr sceptre_user_data.env
 
-  OpenSearchDomainArn: !stack_output dev/regional/opensearch.yaml::OpenSearchDomainArn
+  SsmPath: !stack_attr sceptre_user_data.ssm_path
+  Domain: !stack_attr sceptre_user_data.domain
+
+  # OpenSearchDomainArn: !stack_output dev/regional/opensearch.yaml::OpenSearchDomainArn
 
   EventBusArn: !stack_output dev/regional/eventbridge.yaml::EventBusArn
 
   DmpTableArn: !stack_output dev/regional/dynamo.yaml::DynamoTableArn
+
+  IndexTableArn: !stack_output dev/regional/dynamo-index-table.yaml::DynamoIndexTableArn
diff --git a/config/dev/regional/lambda-vpc.yaml b/config/dev/regional/lambda-vpc.yaml
diff --git a/config/prd/regional/dynamo-index-table.yaml b/config/prd/regional/dynamo-index-table.yaml
@@ -0,0 +1,19 @@
+template:
+  path: dynamo-index-table.yaml
+  type: file
+
+parameters:
+  Env: !stack_attr sceptre_user_data.env
+
+  Domain: !stack_attr sceptre_user_data.domain
+
+  SsmPath: !stack_attr sceptre_user_data.ssm_path
+
+  # Dynamo settings
+  #   See: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html
+  DynamoTableClass: 'STANDARD'
+  DynamoEnableContributorInsights: 'true'
+  DynamoEnablePointInTimeRecovery: 'true'
+  DynamoBillingMode: 'PAY_PER_REQUEST'
+  DynamoReadCapacityUnits: '8'
+  DynamoWriteCapacityUnits: '30'
diff --git a/config/prd/regional/lambda-iam.yaml b/config/prd/regional/lambda-iam.yaml
@@ -5,8 +5,13 @@ template:
 parameters:
   Env: !stack_attr sceptre_user_data.env
 
-  OpenSearchDomainArn: !stack_output prd/regional/opensearch.yaml::OpenSearchDomainArn
+  SsmPath: !stack_attr sceptre_user_data.ssm_path
+  Domain: !stack_attr sceptre_user_data.domain
+
+  # OpenSearchDomainArn: !stack_output prd/regional/opensearch.yaml::OpenSearchDomainArn
 
   EventBusArn: !stack_output prd/regional/eventbridge.yaml::EventBusArn
 
   DmpTableArn: !stack_output prd/regional/dynamo.yaml::DynamoTableArn
+
+  IndexTableArn: !stack_output prd/regional/dynamo-index-table.yaml::DynamoIndexTableArn
diff --git a/config/stg/regional/dynamo-index-table.yaml b/config/stg/regional/dynamo-index-table.yaml
@@ -0,0 +1,19 @@
+template:
+  path: dynamo-index-table.yaml
+  type: file
+
+parameters:
+  Env: !stack_attr sceptre_user_data.env
+
+  Domain: !stack_attr sceptre_user_data.domain
+
+  SsmPath: !stack_attr sceptre_user_data.ssm_path
+
+  # Dynamo settings
+  #   See: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html
+  DynamoTableClass: 'STANDARD'
+  DynamoEnableContributorInsights: 'true'
+  DynamoEnablePointInTimeRecovery: 'true'
+  DynamoBillingMode: 'PAY_PER_REQUEST'
+  DynamoReadCapacityUnits: '8'
+  DynamoWriteCapacityUnits: '30'
diff --git a/config/stg/regional/lambda-iam.yaml b/config/stg/regional/lambda-iam.yaml
@@ -5,8 +5,13 @@ template:
 parameters:
   Env: !stack_attr sceptre_user_data.env
 
-  OpenSearchDomainArn: !stack_output stg/regional/opensearch.yaml::OpenSearchDomainArn
+  SsmPath: !stack_attr sceptre_user_data.ssm_path
+  Domain: !stack_attr sceptre_user_data.domain
+
+  # OpenSearchDomainArn: !stack_output stg/regional/opensearch.yaml::OpenSearchDomainArn
 
   EventBusArn: !stack_output stg/regional/eventbridge.yaml::EventBusArn
 
   DmpTableArn: !stack_output stg/regional/dynamo.yaml::DynamoTableArn
+
+  IndexTableArn: !stack_output stg/regional/dynamo-index-table.yaml::DynamoIndexTableArn
diff --git a/config/stg/regional/lambda-vpc.yaml b/config/stg/regional/lambda-vpc.yaml
diff --git a/src/lambdas/harvesters/datacite/.aws-sam/build.toml b/src/lambdas/harvesters/datacite/.aws-sam/build.toml
@@ -1,6 +1,6 @@
 # This file is auto generated by SAM CLI build command
 
-[function_build_definitions.66e880ba-8a1c-4a71-9af7-3212a8fef2f9]
+[function_build_definitions.d040dc14-b551-4b81-b69b-29ae4079ae63]
 codeuri = "/Users/briley/Documents/workspace/dmsp_aws_prototype/src/lambdas/harvesters/datacite"
 runtime = "ruby3.2"
 architecture = "x86_64"

diff --git a/src/lambdas/harvesters/datacite/.aws-sam/build/Function/sam_build_deploy.rb b/src/lambdas/harvesters/datacite/.aws-sam/build/Function/sam_build_deploy.rb
@@ -51,7 +51,7 @@
   # outputs. The env prefix will be appended to each name your provide.
   #    For example if the name of the parameter is 'DomainName' this script will look for 'dev-DomainName'
   @cf_params = %w[HarvesterRoleArn S3PrivateBucketId BaselineLayerId EventBusArn DynamoTableName
-                  SnsTopicEmailArn DeadLetterQueueArn]
+                  DynamoIndexTableName SnsTopicEmailArn DeadLetterQueueArn]
 
   # List the names of all other parameters whose values should be available as SSM parameters. The name must
   # match the final part of the SSM key name. This script will append the prefix automatically.

diff --git a/src/lambdas/harvesters/datacite/.aws-sam/build/Function/template.yaml b/src/lambdas/harvesters/datacite/.aws-sam/build/Function/template.yaml
@@ -28,6 +28,9 @@ Parameters:
   DynamoTableName:
     Type: 'String'
 
+  DynamoIndexTableName:
+    Type: 'String'
+
   EventBusArn:
     Type: 'String'
 
@@ -46,6 +49,7 @@ Resources:
       Handler: 'app.Functions::DataCiteHarvester.process'
       Runtime: 'ruby3.2'
       Timeout: 600
+      MemorySize: 256
       Architectures:
         - 'x86_64'
       Layers:
@@ -56,9 +60,10 @@ Resources:
           LAMBDA_ENV: !Ref Env
           LOG_LEVEL: !Ref DebugLevel # 'debug'
           DYNAMO_TABLE: !Ref DynamoTableName
+          DYNAMO_INDEX_TABLE: !Ref DynamoIndexTableName
           SNS_FATAL_ERROR_TOPIC: !Ref SnsTopicEmailArn
       Events:
-        DataCiteHarvesting:
+        DataCiteHarvester:
           Type: 'EventBridgeRule'
           Properties:
             DeadLetterConfig:

diff --git a/src/lambdas/harvesters/datacite/.aws-sam/build/template.yaml b/src/lambdas/harvesters/datacite/.aws-sam/build/template.yaml
@@ -19,6 +19,8 @@ Parameters:
     Type: String
   DynamoTableName:
     Type: String
+  DynamoIndexTableName:
+    Type: String
   EventBusArn:
     Type: String
   SnsTopicEmailArn:
@@ -33,6 +35,7 @@ Resources:
       Handler: app.Functions::DataCiteHarvester.process
       Runtime: ruby3.2
       Timeout: 600
+      MemorySize: 256
       Architectures:
       - x86_64
       Layers:
@@ -47,10 +50,12 @@ Resources:
             Ref: DebugLevel
           DYNAMO_TABLE:
             Ref: DynamoTableName
+          DYNAMO_INDEX_TABLE:
+            Ref: DynamoIndexTableName
           SNS_FATAL_ERROR_TOPIC:
             Ref: SnsTopicEmailArn
       Events:
-        DataCiteHarvesting:
+        DataCiteHarvester:
           Type: EventBridgeRule
           Properties:
             DeadLetterConfig: