From 0490411c9f35347d0a9980d10498f5f1b2fb62e5 Mon Sep 17 00:00:00 2001 From: briri Date: Fri, 8 Dec 2023 11:05:53 -0800 Subject: [PATCH 1/3] updates to prd configs and added initial_setup.rb script to create initial SSM parameters --- config/prd/global/route53.yaml | 2 +- config/prd/regional/config.yaml | 5 +- initial_setup.rb | 58 +++++++++++++ templates/dynamo.yaml | 144 ++++++++++++++++---------------- 4 files changed, 132 insertions(+), 77 deletions(-) create mode 100644 initial_setup.rb diff --git a/config/prd/global/route53.yaml b/config/prd/global/route53.yaml index 3770eb2e..5abab048 100644 --- a/config/prd/global/route53.yaml +++ b/config/prd/global/route53.yaml @@ -13,4 +13,4 @@ parameters: # This HostedZone is the default for ALL CloudFront Distributions TargetHostedZoneId: 'Z2FDTNDATAQYW2' - SetIdentifier: 'frobozz' + SetIdentifier: 'quendor' diff --git a/config/prd/regional/config.yaml b/config/prd/regional/config.yaml index c779b4fd..86da18e4 100644 --- a/config/prd/regional/config.yaml +++ b/config/prd/regional/config.yaml @@ -3,14 +3,11 @@ # my_variable: !stack_attr sceptre_user_data.[variable] sceptre_user_data: vpc_id: !stack_output_external cdl-uc3-prd-vpc-stack::vpc + public_subnets: - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2a - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2c - private_subnets: - - !stack_output_external cdl-uc3-prd-privatesubnet-stack::privatesubnet2a - - !stack_output_external cdl-uc3-prd-privatesubnet-stack::privatesubnet2b - - !stack_output_external cdl-uc3-prd-privatesubnet-stack::privatesubnet2c hosted_zone: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId diff --git a/initial_setup.rb b/initial_setup.rb new file mode 100644 index 00000000..a1d194ce --- /dev/null +++ b/initial_setup.rb @@ -0,0 +1,58 @@ +require 'optparse' + +@opts = { override: false, region: 'us-west-2', ezid_debug: false, pause_ezid: false } + +OptionParser.new do |parser| + parser.banner = "Usage: ruby initial_setup [options]" + parser.on("-e", "--environment ENV", "The environment (Example: dev)") { |e| @opts[:env] = e } + parser.on("-r", "--region AWS_REGION", "The AWS region (Default: us-west-2)") { |r| @opts[:region] = r } + parser.on("-o", "--override", TrueClass, "Replace existing values") { |o| @opts[:override] = o } + + parser.on("-a", "--admin-email EMAIL", "The Administrator email address") { |a| @opts[:email] = a } + + parser.on("-x", "--pause-ezid", TrueClass, "Pause EZID submissions") { |o| @opts[:pause_ezid] = o } + + parser.on("-m", "--ezid-debug-mode", TrueClass, "Increase Lambda log output") { |m| @opts[:ezid_debug] = m } + parser.on("-n", "--ezid-orgname NAME", "Your EZID hosting insitution name") { |n| @opts[:ezid_org] = n } + parser.on("-s", "--ezid-shoulder SHOULDER", "Your EZID DOI shoulder") { |s| @opts[:ezid_shoulder] = s } + parser.on("-u", "--ezid-username USER", "Your EZID username") { |u| @opts[:ezid_user] = u } + parser.on("-p", "--ezid-password PWD", "Your EZID password") { |p| @opts[:ezid_pwd] = p } +end.parse! + +def put_param(key:, val:, secure: false, override: false) + name = "/uc3/dmp/hub/#{@opts[:env]}/#{key}" + args = [ + "--region #{@opts[:region]}", + "--name #{name}", + "--value '#{val}'", + "--type #{secure ? 'SecureString' : 'String'}" + ] + args << "--overwrite" if @opts[:override] + + puts "Adding value for SSM parameter #{name} --> '#{key == 'EzidPassword' ? '********' : val}'" + `aws ssm put-parameter #{args.join(' ')}` +end + +if @opts.length > 3 && !@opts[:env].nil? + puts "Using options:" + pp @opts + puts "" + + ezid_doi_base = 'https://doi.org/' + ezid_url = @opts[:env].downcase == 'prd' ? 'https://ezid.cdlib.org/' : 'https://ezid-stg.cdlib.org/' + + put_param(key: 'AdminEmail', val: @opts[:email]) unless @opts[:email].nil? + + put_param(key: 'EzidApiUrl', val: ezid_url) + put_param(key: 'EzidBaseUrl', val: ezid_doi_base) + + put_param(key: 'EzidDebugMode', val: @opts[:ezid_debug]) + put_param(key: 'EzidPaused', val: @opts[:pause_ezid]) + + put_param(key: 'EzidHostingInstitution', val: @opts[:ezid_org]) unless @opts[:ezid_org].nil? + put_param(key: 'EzidShoulder', val: @opts[:ezid_shoulder], secure: true) unless @opts[:ezid_shoulder].nil? + put_param(key: 'EzidUsername', val: @opts[:ezid_user], secure: true) unless @opts[:ezid_user].nil? + put_param(key: 'EzidPassword', val: @opts[:ezid_pwd], secure: true) unless @opts[:ezid_pwd].nil? +else + puts 'You must specify the environment and one or more options! Run `ruby initial_setup -h` for more info.' +end diff --git a/templates/dynamo.yaml b/templates/dynamo.yaml index c4591ad3..6d572080 100644 --- a/templates/dynamo.yaml +++ b/templates/dynamo.yaml @@ -13,8 +13,8 @@ Parameters: SsmPath: Type: 'String' - OpenSearchDomain: - Type: 'String' + # OpenSearchDomain: + # Type: 'String' DynamoTableClass: Type: 'String' @@ -63,61 +63,61 @@ Resources: # DynamoDB polymorphic table for external info (e.g. ROR, DataCite, Crossref, etc.) # See: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html # ---------------------------------------------- - ResourcesDynamoTable: - Type: 'AWS::DynamoDB::Table' - DeletionPolicy: !If - - PreventDelete - - 'Retain' - - 'Delete' - Properties: - KeySchema: - - AttributeName: 'RESOURCE_TYPE' - KeyType: 'HASH' - - AttributeName: 'ID' - KeyType: 'RANGE' - - AttributeDefinitions: - # The type of record (e.g. FUNDER, REPOSITORY, PERSON, etc.) - - AttributeName: 'RESOURCE_TYPE' - AttributeType: 'S' - # The identifier for the resource (e.g. ROR, ORCID, URL, etc.) - - AttributeName: 'ID' - AttributeType: 'S' - # The last updated date - - AttributeName: 'source' - AttributeType: 'S' - - GlobalSecondaryIndexes: - # An index to help us quickly fetch all of the - - IndexName: 'source_gsi' - KeySchema: - - AttributeName: 'source' - KeyType: 'HASH' - - AttributeName: 'ID' - KeyType: 'RANGE' - Projection: - ProjectionType: 'INCLUDE' - NonKeyAttributes: ['RESOURCE_TYPE', 'touch_date'] - ContributorInsightsSpecification: - Enabled: !Ref DynamoEnableContributorInsights - ProvisionedThroughput: !If - - ProvisionedCapacity - - ReadCapacityUnits: !Ref DynamoReadCapacityUnits - WriteCapacityUnits: !Ref DynamoWriteCapacityUnits - - !Ref AWS::NoValue - - BillingMode: !Ref DynamoBillingMode - TableClass: !Ref DynamoTableClass - ProvisionedThroughput: !If - - ProvisionedCapacity - - ReadCapacityUnits: !Ref DynamoReadCapacityUnits - WriteCapacityUnits: !Ref DynamoWriteCapacityUnits - - !Ref AWS::NoValue - ContributorInsightsSpecification: - Enabled: !Ref DynamoEnableContributorInsights - # Determine cost before enabling - PointInTimeRecoverySpecification: - PointInTimeRecoveryEnabled: !Ref DynamoEnablePointInTimeRecovery + # ResourcesDynamoTable: + # Type: 'AWS::DynamoDB::Table' + # DeletionPolicy: !If + # - PreventDelete + # - 'Retain' + # - 'Delete' + # Properties: + # KeySchema: + # - AttributeName: 'RESOURCE_TYPE' + # KeyType: 'HASH' + # - AttributeName: 'ID' + # KeyType: 'RANGE' + # + # AttributeDefinitions: + # # The type of record (e.g. FUNDER, REPOSITORY, PERSON, etc.) + # - AttributeName: 'RESOURCE_TYPE' + # AttributeType: 'S' + # # The identifier for the resource (e.g. ROR, ORCID, URL, etc.) + # - AttributeName: 'ID' + # AttributeType: 'S' + # # The last updated date + # - AttributeName: 'source' + # AttributeType: 'S' + # + # GlobalSecondaryIndexes: + # # An index to help us quickly fetch all of the + # - IndexName: 'source_gsi' + # KeySchema: + # - AttributeName: 'source' + # KeyType: 'HASH' + # - AttributeName: 'ID' + # KeyType: 'RANGE' + # Projection: + # ProjectionType: 'INCLUDE' + # NonKeyAttributes: ['RESOURCE_TYPE', 'touch_date'] + # ContributorInsightsSpecification: + # Enabled: !Ref DynamoEnableContributorInsights + # ProvisionedThroughput: !If + # - ProvisionedCapacity + # - ReadCapacityUnits: !Ref DynamoReadCapacityUnits + # WriteCapacityUnits: !Ref DynamoWriteCapacityUnits + # - !Ref AWS::NoValue + # + # BillingMode: !Ref DynamoBillingMode + # TableClass: !Ref DynamoTableClass + # ProvisionedThroughput: !If + # - ProvisionedCapacity + # - ReadCapacityUnits: !Ref DynamoReadCapacityUnits + # WriteCapacityUnits: !Ref DynamoWriteCapacityUnits + # - !Ref AWS::NoValue + # ContributorInsightsSpecification: + # Enabled: !Ref DynamoEnableContributorInsights + # # Determine cost before enabling + # PointInTimeRecoverySpecification: + # PointInTimeRecoveryEnabled: !Ref DynamoEnablePointInTimeRecovery # ---------------------------------------------- # DynamoDB polymorphic table for DMPs @@ -254,19 +254,19 @@ Resources: Type: 'String' Value: !Select [1, !Split ['/', !GetAtt DynamoTable.Arn]] - OpenSearchDomainParameter: - Type: 'AWS::SSM::Parameter' - Properties: - Description: !Sub "${AWS::StackName} OpenSearch Domain" - Name: !Sub "${SsmPath}OpenSearchDomain" - Type: 'String' - Value: !Ref OpenSearchDomain + # OpenSearchDomainParameter: + # Type: 'AWS::SSM::Parameter' + # Properties: + # Description: !Sub "${AWS::StackName} OpenSearch Domain" + # Name: !Sub "${SsmPath}OpenSearchDomain" + # Type: 'String' + # Value: !Ref OpenSearchDomain Outputs: - ResourcesDynamoTableName: - Value: !Select [1, !Split ['/', !GetAtt ResourcesDynamoTable.Arn]] - Export: - Name: !Sub '${Env}-ResourcesDynamoTableName' + # ResourcesDynamoTableName: + # Value: !Select [1, !Split ['/', !GetAtt ResourcesDynamoTable.Arn]] + # Export: + # Name: !Sub '${Env}-ResourcesDynamoTableName' DynamoTableName: Value: !Select [1, !Split ['/', !GetAtt DynamoTable.Arn]] @@ -286,7 +286,7 @@ Outputs: Export: Name: !Sub '${Env}-DynamoTableArn' - ResourcesDynamoTableArn: - Value: !GetAtt ResourcesDynamoTable.Arn - Export: - Name: !Sub '${Env}-ResourcesDynamoTableArn' \ No newline at end of file + # ResourcesDynamoTableArn: + # Value: !GetAtt ResourcesDynamoTable.Arn + # Export: + # Name: !Sub '${Env}-ResourcesDynamoTableArn' \ No newline at end of file From 8bf0bafb5dcb8c0b60dabe9513e4eadf37031fdc Mon Sep 17 00:00:00 2001 From: briri Date: Mon, 18 Dec 2023 15:20:12 -0800 Subject: [PATCH 2/3] attempt to pull in hostedzone --- .DS_Store | Bin 6148 -> 6148 bytes config/prd/config.yaml | 2 +- config/prd/global/cert.yaml | 2 +- config/prd/global/config.yaml | 1 - config/prd/global/route53.yaml | 2 +- config/prd/regional/config.yaml | 2 +- config/stg/global/cert.yaml | 2 +- config/stg/global/route53.yaml | 2 +- config/stg/regional/config.yaml | 2 +- 9 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.DS_Store b/.DS_Store index c342d2b84576747623ff4aea189dbabe167c8877..a1dc93a79e3f55e211bec07896dbf63910cee137 100644 GIT binary patch delta 49 ycmZoMXfc@J&&V<{z#2&OScx+L0VhKVLn=cqLjgk$Ln4@6ym6lv`(}2Izx)6rgA5`7 delta 28 icmZoMXfc@J&&WJ6z#2&OSZ#cF2pelt7AUw#0BDG34q diff --git a/config/prd/config.yaml b/config/prd/config.yaml index a739aa13..6d98ec86 100644 --- a/config/prd/config.yaml +++ b/config/prd/config.yaml @@ -14,4 +14,4 @@ stack_tags: Contact: 'briley' # In the cdl-uc3-prd account devs must call cloudformation using a service role -cloudformation_service_role: 'arn:aws:iam::834750697783:role/uc3-prd-ops-cfn-service-role' +cloudformation_service_role: !ssm_parameter /uc3/prd/CfnServiceRoleArn diff --git a/config/prd/global/cert.yaml b/config/prd/global/cert.yaml index 5f8c50c2..34beba30 100644 --- a/config/prd/global/cert.yaml +++ b/config/prd/global/cert.yaml @@ -3,7 +3,7 @@ template: type: file parameters: - HostedZoneId: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId + HostedZoneId: !stack_attr sceptre_user_data.hosted_zone Domain: !stack_attr sceptre_user_data.domain diff --git a/config/prd/global/config.yaml b/config/prd/global/config.yaml index c4a5cae7..c2f34e37 100644 --- a/config/prd/global/config.yaml +++ b/config/prd/global/config.yaml @@ -6,5 +6,4 @@ region: 'us-east-1' sceptre_user_data: env: 'prd' domain: 'dmphub.uc3prd.cdlib.net' - hosted_zone: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId ssm_path: '/uc3/dmp/hub/prd/' diff --git a/config/prd/global/route53.yaml b/config/prd/global/route53.yaml index 5abab048..cc5ac1fd 100644 --- a/config/prd/global/route53.yaml +++ b/config/prd/global/route53.yaml @@ -3,7 +3,7 @@ template: type: 'file' parameters: - HostedZoneId: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId + HostedZoneId: !stack_attr sceptre_user_data.hosted_zone Env: !stack_attr sceptre_user_data.env Domain: !stack_attr sceptre_user_data.domain diff --git a/config/prd/regional/config.yaml b/config/prd/regional/config.yaml index 86da18e4..1e79b5e2 100644 --- a/config/prd/regional/config.yaml +++ b/config/prd/regional/config.yaml @@ -9,7 +9,7 @@ sceptre_user_data: - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2c - hosted_zone: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId + # hosted_zone: !stack_output_external uc3-ops-aws-prd-route53::HostedZoneIdUc3PrdCdlibNet env: 'prd' domain: 'dmphub.uc3prd.cdlib.net' diff --git a/config/stg/global/cert.yaml b/config/stg/global/cert.yaml index 3ca0a8c1..34beba30 100644 --- a/config/stg/global/cert.yaml +++ b/config/stg/global/cert.yaml @@ -3,7 +3,7 @@ template: type: file parameters: - HostedZoneId: !ssm_parameter /uc3/dmp/hub/stg/HostedZoneId + HostedZoneId: !stack_attr sceptre_user_data.hosted_zone Domain: !stack_attr sceptre_user_data.domain diff --git a/config/stg/global/route53.yaml b/config/stg/global/route53.yaml index ad632ff5..6d7e84cf 100644 --- a/config/stg/global/route53.yaml +++ b/config/stg/global/route53.yaml @@ -3,7 +3,7 @@ template: type: 'file' parameters: - HostedZoneId: !ssm_parameter /uc3/dmp/hub/stg/HostedZoneId + HostedZoneId: !stack_attr sceptre_user_data.hosted_zone Env: !stack_attr sceptre_user_data.env Domain: !stack_attr sceptre_user_data.domain diff --git a/config/stg/regional/config.yaml b/config/stg/regional/config.yaml index 291bd5ad..c86e463d 100644 --- a/config/stg/regional/config.yaml +++ b/config/stg/regional/config.yaml @@ -12,7 +12,7 @@ sceptre_user_data: - !stack_output_external cdl-uc3-prd-privatesubnet-stack::privatesubnet2b - !stack_output_external cdl-uc3-prd-privatesubnet-stack::privatesubnet2c - hosted_zone: !ssm_parameter /uc3/dmp/hub/stg/HostedZoneId + hosted_zone: !stack_output_external uc3-ops-aws-prd-route53::HostedZoneIdUc3StgCdlibNet env: 'stg' domain: 'dmphub.uc3stg.cdlib.net' From adc98c684d2dd8f5c6eeff5b8b999105eb5aee59 Mon Sep 17 00:00:00 2001 From: briri Date: Tue, 19 Dec 2023 14:14:31 -0800 Subject: [PATCH 3/3] updates to fix issues with prod deploy (most name conflicts since stg/prd are in same env) --- config/prd/config.yaml | 1 + config/prd/global/config.yaml | 1 + config/prd/regional/config.yaml | 2 +- config/prd/regional/dynamo.yaml | 2 +- initial_setup.rb | 2 ++ src/sam/template.yaml | 10 +++++----- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config/prd/config.yaml b/config/prd/config.yaml index 6d98ec86..5371c788 100644 --- a/config/prd/config.yaml +++ b/config/prd/config.yaml @@ -2,6 +2,7 @@ # provide the name of an existing bucket. # template_bucket_name: 'uc3-dmp-hub-cf-bucket-cdluc3prd' + # Stack tags are attached to every AWS resource created by these CloudFormation templates # with the exception of the Lambda functions and layers which are managed by AWS SAM. # diff --git a/config/prd/global/config.yaml b/config/prd/global/config.yaml index c2f34e37..c4a5cae7 100644 --- a/config/prd/global/config.yaml +++ b/config/prd/global/config.yaml @@ -6,4 +6,5 @@ region: 'us-east-1' sceptre_user_data: env: 'prd' domain: 'dmphub.uc3prd.cdlib.net' + hosted_zone: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId ssm_path: '/uc3/dmp/hub/prd/' diff --git a/config/prd/regional/config.yaml b/config/prd/regional/config.yaml index 1e79b5e2..86da18e4 100644 --- a/config/prd/regional/config.yaml +++ b/config/prd/regional/config.yaml @@ -9,7 +9,7 @@ sceptre_user_data: - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2b - !stack_output_external cdl-uc3-prd-defaultsubnet-stack::defaultsubnet2c - # hosted_zone: !stack_output_external uc3-ops-aws-prd-route53::HostedZoneIdUc3PrdCdlibNet + hosted_zone: !ssm_parameter /uc3/dmp/hub/prd/HostedZoneId env: 'prd' domain: 'dmphub.uc3prd.cdlib.net' diff --git a/config/prd/regional/dynamo.yaml b/config/prd/regional/dynamo.yaml index d6b383d3..521a45ad 100644 --- a/config/prd/regional/dynamo.yaml +++ b/config/prd/regional/dynamo.yaml @@ -31,4 +31,4 @@ hooks: # Build and deploy the React based DMP ID Landing Page to the CloudFront S3 bucket # Args are: environment - - !cmd 'cd ./src/landing_page && ruby build_deply.sh prd' + - !cmd 'cd ./src/landing_page && ruby build_deply.rb prd' diff --git a/initial_setup.rb b/initial_setup.rb index a1d194ce..9f31db75 100644 --- a/initial_setup.rb +++ b/initial_setup.rb @@ -8,6 +8,7 @@ parser.on("-r", "--region AWS_REGION", "The AWS region (Default: us-west-2)") { |r| @opts[:region] = r } parser.on("-o", "--override", TrueClass, "Replace existing values") { |o| @opts[:override] = o } + parser.on("-h", "--hosted-zone ZONE", "The HostedZoneId") { |h| @opts[:hosted_zone] = h } parser.on("-a", "--admin-email EMAIL", "The Administrator email address") { |a| @opts[:email] = a } parser.on("-x", "--pause-ezid", TrueClass, "Pause EZID submissions") { |o| @opts[:pause_ezid] = o } @@ -41,6 +42,7 @@ def put_param(key:, val:, secure: false, override: false) ezid_doi_base = 'https://doi.org/' ezid_url = @opts[:env].downcase == 'prd' ? 'https://ezid.cdlib.org/' : 'https://ezid-stg.cdlib.org/' + put_param(key: 'HostedZoneId', val: @opts[:hosted_zone]) unless @opts[:hosted_zone].nil? put_param(key: 'AdminEmail', val: @opts[:email]) unless @opts[:email].nil? put_param(key: 'EzidApiUrl', val: ezid_url) diff --git a/src/sam/template.yaml b/src/sam/template.yaml index 9dfc7951..85ea8522 100644 --- a/src/sam/template.yaml +++ b/src/sam/template.yaml @@ -103,7 +103,7 @@ Resources: Type: 'AWS::IAM::ManagedPolicy' Properties: Description: 'DMPHub - Read access to the DynamoDB Table' - ManagedPolicyName: 'DmpHubDynamoReadPolicy' + ManagedPolicyName: !Sub 'DmpHub${Env}DynamoReadPolicy' PolicyDocument: Version: '2012-10-17' Statement: @@ -124,7 +124,7 @@ Resources: Type: 'AWS::IAM::ManagedPolicy' Properties: Description: 'DMPHub - Write access (and GetItem) to the DynamoDB Table' - ManagedPolicyName: 'DmpHubDynamoWritePolicy' + ManagedPolicyName: !Sub 'DmpHub${Env}DynamoWritePolicy' PolicyDocument: Version: '2012-10-17' Statement: @@ -205,7 +205,7 @@ Resources: Type: 'AWS::IAM::ManagedPolicy' Properties: Description: 'DMPHub - Access to the EventBus' - ManagedPolicyName: 'DmpHubEventBusPolicy' + ManagedPolicyName: !Sub 'DmpHub${Env}EventBusPolicy' PolicyDocument: Version: '2012-10-17' Statement: @@ -218,7 +218,7 @@ Resources: Type: 'AWS::IAM::ManagedPolicy' Properties: Description: 'DMPHub - Access to the Cognito User Pool Client' - ManagedPolicyName: 'DmpHubCognitoPolicy' + ManagedPolicyName: !Sub 'DmpHub${Env}CognitoPolicy' PolicyDocument: Version: '2012-10-17' Statement: @@ -231,7 +231,7 @@ Resources: Type: 'AWS::IAM::ManagedPolicy' Properties: Description: 'DMPHub - Access to the S3 Bucket' - ManagedPolicyName: 'DmpHubS3Policy' + ManagedPolicyName: !Sub 'DmpHub${Env}S3Policy' PolicyDocument: Version: '2012-10-17' Statement: