From 81ae5cdb1ec04adcb7620edabc515c32d71070b0 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Fri, 16 Nov 2018 15:11:54 +0000 Subject: [PATCH 01/12] keep the list of valid autoscaling termination policies up to date --- modules/Gemfile.lock | 8 ++++---- modules/mu/clouds/aws/server_pool.rb | 17 +++++++++++++++++ modules/mu/config/server_pool.rb | 9 --------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index d46d7bc99..2efcf5223 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -12,7 +12,7 @@ GEM specs: addressable (2.4.0) ast (2.4.0) - aws-sdk-core (2.11.162) + aws-sdk-core (2.11.167) aws-sigv4 (~> 1.0) jmespath (~> 1.0) aws-sigv4 (1.0.3) @@ -96,7 +96,7 @@ GEM coderay (1.1.2) color (1.8) colorize (0.8.1) - concurrent-ruby (1.0.5) + concurrent-ruby (1.1.3) cucumber-core (4.0.0) backports (>= 3.8.0) cucumber-tag_expressions (~> 1.1.0) @@ -250,10 +250,10 @@ GEM polyglot (0.3.5) powerpack (0.1.2) proxifier (1.0.3) - pry (0.11.3) + pry (0.12.0) coderay (~> 1.1.0) method_source (~> 0.9.0) - rack (2.0.5) + rack (2.0.6) rainbow (3.0.0) rake (12.3.1) representable (3.0.4) diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index f3d7f9160..188d0714b 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -342,6 +342,7 @@ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: n # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource def self.schema(config) toplevel_required = [] + schema = { "generate_iam_role" => { "type" => "boolean", @@ -399,6 +400,15 @@ def self.schema(config) } } }, + "termination_policies" => { + "type" => "array", + "minItems" => 1, + "items" => { + "type" => "String", + "default" => "Default", + "enum" => MU::Cloud::AWS.autoscale.describe_termination_policy_types.termination_policy_types + } + }, "ingress_rules" => { "items" => { "properties" => { @@ -430,6 +440,13 @@ def self.schema(config) def self.validateConfig(pool, configurator) ok = true + if pool["termination_policy"] + valid_policies = MU::Cloud::AWS.autoscale(pool['region']).describe_termination_policy_types.termination_policy_types + if !valid_policies.include?(pool["termination_policy"]) + ok = false + MU.log "Termination policy #{pool["termination_policy"]} is not valid in region #{pool['region']}", MU::ERR, details: valid_policies + end + end if !pool["schedule"].nil? pool["schedule"].each { |s| diff --git a/modules/mu/config/server_pool.rb b/modules/mu/config/server_pool.rb index fe2f0f407..7667c08b8 100644 --- a/modules/mu/config/server_pool.rb +++ b/modules/mu/config/server_pool.rb @@ -144,15 +144,6 @@ def self.schema } } }, - "termination_policies" => { - "type" => "array", - "minItems" => 1, - "items" => { - "type" => "String", - "default" => "Default", - "enum" => ["Default", "OldestInstance", "NewestInstance", "OldestLaunchConfiguration", "ClosestToNextInstanceHour"] - } - }, #XXX this needs its own primitive and discovery mechanism "zones" => { "type" => "array", From 01d26200c4881af2d7eb3ca889aaf47075c376af Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Fri, 16 Nov 2018 17:49:13 +0000 Subject: [PATCH 02/12] scale_in_protection --- modules/mu/clouds/aws/server_pool.rb | 75 +++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index 188d0714b..302084cb4 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -203,11 +203,60 @@ def create max_size: @config['max_size'] ) end + + if @config['scale_in_protection'] + need_instances = @config['scale_in_protection'].match(/^\d+$/) ? @config['scale_in_protection'].to_i : @config['min_size'] + setScaleInProtection(need_instances) + end + MU.log "See /var/log/mu-momma-cat.log for asynchronous bootstrap progress.", MU::NOTICE return asg end + # Make sure we have a set of instances with scale-in protection set which jives with our config + # @param need_instances [Integer]: The number of instanceswhich must have scale-in protection set + def setScaleInProtection(need_instances = @config['min_size']) + live_instances = [] + begin + desc = MU::Cloud::AWS.autoscale(@config['region']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first + + live_instances = desc.instances.map { |i| i.instance_id } + already_set = 0 + desc.instances.each { |i| + already_set += 1 if i.protected_from_scale_in + } + if live_instances.size < need_instances + sleep 5 + elsif already_set > need_instances + unset_me = live_instances.sample(already_set - need_instances) + MU.log "Disabling scale-in protection for #{unset_me.size.to_s} instances in #{@mu_name}", MU::NOTICE, details: unset_me + MU::Cloud::AWS.autoscale(@config['region']).set_instance_protection( + auto_scaling_group_name: @mu_name, + instance_ids: unset_me, + protected_from_scale_in: false + ) + elsif already_set < need_instances + live_instances = live_instances.sample(need_instances) + MU.log "Enabling scale-in protection for #{@config['scale_in_protection']} instances in #{@mu_name}", details: live_instances + begin + MU::Cloud::AWS.autoscale(@config['region']).set_instance_protection( + auto_scaling_group_name: @mu_name, + instance_ids: live_instances, + protected_from_scale_in: true + ) + rescue Aws::AutoScaling::Errors::ValidationError => e + if e.message.match(/not in InService/i) + sleep 5 + retry + else + raise e + end + end + end + end while live_instances.size < need_instances + end + # List out the nodes that are members of this pool # @return [Array] def listNodes @@ -290,6 +339,7 @@ def groom asg_options.delete(:tags) asg_options[:min_size] = @config["min_size"] asg_options[:max_size] = @config["max_size"] + asg_options[:new_instances_protected_from_scale_in] = (@config['scale_in_protection'] == "all") if asg_options[:target_group_arns] MU::Cloud::AWS.autoscale(@config['region']).attach_load_balancer_target_groups( auto_scaling_group_name: @mu_name, @@ -300,6 +350,18 @@ def groom MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(asg_options) + if @config['scale_in_protection'] + if @config['scale_in_protection'] == "all" + setScaleInProtection(listNodes.size) + elsif @config['scale_in_protection'] == "initial" + setScaleInProtection(@config['min_size']) + elsif @config['scale_in_protection'].match(/^\d+$/) + setScaleInProtection(@config['scale_in_protection'].to_i) + end + else + setScaleInProtection(0) + end + end # Retrieve the AWS descriptor for this Autoscale group @@ -400,6 +462,11 @@ def self.schema(config) } } }, + "scale_in_protection" => { + "type" => "string", + "description" => "Protect instances from scale-in termination. Can be 'all', 'initial' (essentially 'min_size'), or an number; note the number needs to be a string, so put it in quotes", + "pattern" => "^(all|initial|\\d+)$" + }, "termination_policies" => { "type" => "array", "minItems" => 1, @@ -820,11 +887,15 @@ def createUpdateLaunchConfig MU::Cloud::AWS::Server.addStdPoliciesToIAMProfile(@config['iam_role'], region: @config['region']) end + lc_attempts = 0 begin MU::Cloud::AWS.autoscale(@config['region']).create_launch_configuration(launch_options) rescue Aws::AutoScaling::Errors::ValidationError => e - MU.log e.message, MU::WARN - sleep 10 + if lc_attempts > 3 + MU.log "Got error while creating #{@mu_name} Launch Config: #{e.message}, retrying in 10s", MU::WARN + end + sleep 5 + lc_attempts += 1 retry end From 0282f1be034cba9902c3528f4b4dcd7813959e9d Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Fri, 16 Nov 2018 18:59:55 +0000 Subject: [PATCH 03/12] make sure scaling_policies get put on groom --- modules/mu/clouds/aws/server_pool.rb | 123 ++++++++++++++------------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index 302084cb4..7098074dc 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -79,67 +79,6 @@ def create @cloud_id = @mu_name - if @config["scaling_policies"] and @config["scaling_policies"].size > 0 - @config["scaling_policies"].each { |policy| - policy_params = { - :auto_scaling_group_name => @mu_name, - :policy_name => @deploy.getResourceName("#{@config['name']}-#{policy['name']}"), - :adjustment_type => policy['type'], - :policy_type => policy['policy_type'] - } - - if policy["policy_type"] == "SimpleScaling" - policy_params[:cooldown] = policy['cooldown'] - policy_params[:scaling_adjustment] = policy['adjustment'] - elsif policy["policy_type"] == "StepScaling" - step_adjustments = [] - policy['step_adjustments'].each{|step| - step_adjustments << {:metric_interval_lower_bound => step["lower_bound"], :metric_interval_upper_bound => step["upper_bound"], :scaling_adjustment => step["adjustment"]} - } - policy_params[:metric_aggregation_type] = policy['metric_aggregation_type'] - policy_params[:step_adjustments] = step_adjustments - policy_params[:estimated_instance_warmup] = policy['estimated_instance_warmup'] - end - - policy_params[:min_adjustment_magnitude] = policy['min_adjustment_magnitude'] if !policy['min_adjustment_magnitude'].nil? - resp = MU::Cloud::AWS.autoscale(@config['region']).put_scaling_policy(policy_params) - - # If we are creating alarms for scaling policies we need to have the autoscaling policy ARN - # To make life easier we're creating the alarms here - if policy.has_key?("alarms") && !policy["alarms"].empty? - policy["alarms"].each { |alarm| - alarm["alarm_actions"] = [] if !alarm.has_key?("alarm_actions") - alarm["ok_actions"] = [] if !alarm.has_key?("ok_actions") - alarm["alarm_actions"] << resp.policy_arn - alarm["dimensions"] = [{name: "AutoScalingGroupName", value: asg_options[:auto_scaling_group_name]}] - - if alarm["enable_notifications"] - topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"]) - MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"]) - alarm["alarm_actions"] << topic_arn - alarm["ok_actions"] << topic_arn - end - - MU::Cloud::AWS::Alarm.setAlarm( - name: "#{MU.deploy_id}-#{alarm["name"]}".upcase, - ok_actions: alarm["ok_actions"], - alarm_actions: alarm["alarm_actions"], - insufficient_data_actions: alarm["no_data_actions"], - metric_name: alarm["metric_name"], - namespace: alarm["namespace"], - statistic: alarm["statistic"], - dimensions: alarm["dimensions"], - period: alarm["period"], - unit: alarm["unit"], - evaluation_periods: alarm["evaluation_periods"], - threshold: alarm["threshold"], - comparison_operator: alarm["comparison_operator"], - region: @config["region"] - ) - } - end - } - end # Wait and see if we successfully bring up some instances attempts = 0 @@ -362,6 +301,68 @@ def groom setScaleInProtection(0) end + if @config["scaling_policies"] and @config["scaling_policies"].size > 0 + @config["scaling_policies"].each { |policy| + policy_params = { + :auto_scaling_group_name => @mu_name, + :policy_name => @deploy.getResourceName("#{@config['name']}-#{policy['name']}"), + :adjustment_type => policy['type'], + :policy_type => policy['policy_type'] + } + + if policy["policy_type"] == "SimpleScaling" + policy_params[:cooldown] = policy['cooldown'] + policy_params[:scaling_adjustment] = policy['adjustment'] + elsif policy["policy_type"] == "StepScaling" + step_adjustments = [] + policy['step_adjustments'].each{|step| + step_adjustments << {:metric_interval_lower_bound => step["lower_bound"], :metric_interval_upper_bound => step["upper_bound"], :scaling_adjustment => step["adjustment"]} + } + policy_params[:metric_aggregation_type] = policy['metric_aggregation_type'] + policy_params[:step_adjustments] = step_adjustments + policy_params[:estimated_instance_warmup] = policy['estimated_instance_warmup'] + end + + policy_params[:min_adjustment_magnitude] = policy['min_adjustment_magnitude'] if !policy['min_adjustment_magnitude'].nil? + resp = MU::Cloud::AWS.autoscale(@config['region']).put_scaling_policy(policy_params) + + # If we are creating alarms for scaling policies we need to have the autoscaling policy ARN + # To make life easier we're creating the alarms here + if policy.has_key?("alarms") && !policy["alarms"].empty? + policy["alarms"].each { |alarm| + alarm["alarm_actions"] = [] if !alarm.has_key?("alarm_actions") + alarm["ok_actions"] = [] if !alarm.has_key?("ok_actions") + alarm["alarm_actions"] << resp.policy_arn + alarm["dimensions"] = [{name: "AutoScalingGroupName", value: asg_options[:auto_scaling_group_name]}] + + if alarm["enable_notifications"] + topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"]) + MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"]) + alarm["alarm_actions"] << topic_arn + alarm["ok_actions"] << topic_arn + end + + MU::Cloud::AWS::Alarm.setAlarm( + name: "#{MU.deploy_id}-#{alarm["name"]}".upcase, + ok_actions: alarm["ok_actions"], + alarm_actions: alarm["alarm_actions"], + insufficient_data_actions: alarm["no_data_actions"], + metric_name: alarm["metric_name"], + namespace: alarm["namespace"], + statistic: alarm["statistic"], + dimensions: alarm["dimensions"], + period: alarm["period"], + unit: alarm["unit"], + evaluation_periods: alarm["evaluation_periods"], + threshold: alarm["threshold"], + comparison_operator: alarm["comparison_operator"], + region: @config["region"] + ) + } + end + } + end + end # Retrieve the AWS descriptor for this Autoscale group From 346865020a46b57edbb2d7e5e9712c1c15a38530 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Fri, 16 Nov 2018 20:50:00 +0000 Subject: [PATCH 04/12] Autoscale: new scaling policy type, TargetTrackingScaling --- modules/mu/clouds/aws/server_pool.rb | 201 ++++++++++++++++++++++++++- modules/mu/config/alarm.rb | 1 - modules/mu/config/server_pool.rb | 75 ---------- 3 files changed, 200 insertions(+), 77 deletions(-) diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index 7098074dc..b0125e21d 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -279,11 +279,13 @@ def groom asg_options[:min_size] = @config["min_size"] asg_options[:max_size] = @config["max_size"] asg_options[:new_instances_protected_from_scale_in] = (@config['scale_in_protection'] == "all") + tg_arns = [] if asg_options[:target_group_arns] MU::Cloud::AWS.autoscale(@config['region']).attach_load_balancer_target_groups( auto_scaling_group_name: @mu_name, target_group_arns: asg_options[:target_group_arns] ) + tg_arns = asg_options[:target_group_arns].dup asg_options.delete(:target_group_arns) end @@ -306,13 +308,41 @@ def groom policy_params = { :auto_scaling_group_name => @mu_name, :policy_name => @deploy.getResourceName("#{@config['name']}-#{policy['name']}"), - :adjustment_type => policy['type'], :policy_type => policy['policy_type'] } if policy["policy_type"] == "SimpleScaling" policy_params[:cooldown] = policy['cooldown'] policy_params[:scaling_adjustment] = policy['adjustment'] + policy_params[:adjustment_type] = policy['type'] + elsif policy["policy_type"] == "TargetTrackingScaling" + def strToSym(hash) + newhash = {} + hash.each_pair { |k, v| + if v.is_a?(Hash) + newhash[k.to_sym] = strToSym(v) + else + newhash[k.to_sym] = v + end + } + newhash + end + policy_params[:target_tracking_configuration] = strToSym(policy['target_tracking_configuration']) + if policy_params[:target_tracking_configuration][:predefined_metric_specification] and + policy_params[:target_tracking_configuration][:predefined_metric_specification][:predefined_metric_type] == "ALBRequestCountPerTarget" + lb_path = nil + lb = @deploy.deployment["loadbalancers"].values.first + if @deploy.deployment["loadbalancers"].size > 1 + MU.log "Multiple load balancers attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN + end + if lb["targetgroups"].size > 1 + MU.log "Multiple target groups attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN + end + lb_path = lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"].values.first.split(/:/)[5] + + policy_params[:target_tracking_configuration][:predefined_metric_specification][:resource_label] = lb_path + end + policy_params[:estimated_instance_warmup] = policy['estimated_instance_warmup'] elsif policy["policy_type"] == "StepScaling" step_adjustments = [] policy['step_adjustments'].each{|step| @@ -321,9 +351,12 @@ def groom policy_params[:metric_aggregation_type] = policy['metric_aggregation_type'] policy_params[:step_adjustments] = step_adjustments policy_params[:estimated_instance_warmup] = policy['estimated_instance_warmup'] + policy_params[:adjustment_type] = policy['type'] end policy_params[:min_adjustment_magnitude] = policy['min_adjustment_magnitude'] if !policy['min_adjustment_magnitude'].nil? + pp policy_params + # XXX remove before put resp = MU::Cloud::AWS.autoscale(@config['region']).put_scaling_policy(policy_params) # If we are creating alarms for scaling policies we need to have the autoscaling policy ARN @@ -477,6 +510,154 @@ def self.schema(config) "enum" => MU::Cloud::AWS.autoscale.describe_termination_policy_types.termination_policy_types } }, + "scaling_policies" => { + "type" => "array", + "minItems" => 1, + "items" => { + "type" => "object", + "required" => ["name"], + "additionalProperties" => false, + "description" => "A custom AWS Autoscale scaling policy for this pool.", + "properties" => { + "name" => { + "type" => "string" + }, + "alarms" => MU::Config::Alarm.inline, + "type" => { + "type" => "string", + "enum" => ["ChangeInCapacity", "ExactCapacity", "PercentChangeInCapacity"], + "description" => "Specifies whether 'adjustment' is an absolute number or a percentage of the current capacity for SimpleScaling and StepScaling. Valid values are ChangeInCapacity, ExactCapacity, and PercentChangeInCapacity." + }, + "adjustment" => { + "type" => "integer", + "description" => "The number of instances by which to scale. 'type' determines the interpretation of this number (e.g., as an absolute number or as a percentage of the existing Auto Scaling group size). A positive increment adds to the current capacity and a negative value removes from the current capacity. Used only when policy_type is set to 'SimpleScaling'" + }, + "cooldown" => { + "type" => "integer", + "default" => 1, + "description" => "The amount of time, in seconds, after a scaling activity completes and before the next scaling activity can start." + }, + "min_adjustment_magnitude" => { + "type" => "integer", + "description" => "Used when 'type' is set to 'PercentChangeInCapacity', the scaling policy changes the DesiredCapacity of the Auto Scaling group by at least the number of instances specified in the value." + }, + "policy_type" => { + "type" => "string", + "enum" => ["SimpleScaling", "StepScaling", "TargetTrackingScaling"], + "description" => "'StepScaling' will add capacity based on the magnitude of the alarm breach, 'SimpleScaling' will add capacity based on the 'adjustment' value provided. Defaults to 'SimpleScaling'.", + "default" => "SimpleScaling" + }, + "metric_aggregation_type" => { + "type" => "string", + "enum" => ["Minimum", "Maximum", "Average"], + "description" => "Defaults to 'Average' if not specified. Required when policy_type is set to 'StepScaling'", + "default" => "Average" + }, + "step_adjustments" => { + "type" => "array", + "minItems" => 1, + "items" => { + "type" => "object", + "title" => "admin", + "description" => "Requires policy_type 'StepScaling'", + "required" => ["adjustment"], + "additionalProperties" => false, + "properties" => { + "adjustment" => { + "type" => "integer", + "description" => "The number of instances by which to scale at this specific step. Postive value when adding capacity, negative value when removing capacity" + }, + "lower_bound" => { + "type" => "integer", + "description" => "The lower bound value in percentage points above/below the alarm threshold at which to add/remove capacity for this step. Positive value when adding capacity and negative when removing capacity. If this is the first step and capacity is being added this value will most likely be 0" + }, + "upper_bound" => { + "type" => "integer", + "description" => "The upper bound value in percentage points above/below the alarm threshold at which to add/remove capacity for this step. Positive value when adding capacity and negative when removing capacity. If this is the first step and capacity is being removed this value will most likely be 0" + } + } + } + }, + "estimated_instance_warmup" => { + "type" => "integer", + "description" => "Required when policy_type is set to 'StepScaling'" + }, + "target_tracking_configuration" => { + "type" => "object", + "description" => "Required when policy_type is set to 'TargetTrackingScaling' https://docs.aws.amazon.com/sdkforruby/api/Aws/AutoScaling/Types/TargetTrackingConfiguration.html", + "required" => ["target_value"], + "additionalProperties" => false, + "properties" => { + "target_value" => { + "type" => "float", + "description" => "The target value for the metric." + }, + "disable_scale_in" => { + "type" => "boolean", + "description" => "If set to true, new instances created by this policy will not be subject to termination by scaling in.", + "default" => false + }, + "predefined_metric_specification" => { + "type" => "object", + "description" => "A predefined metric. You can specify either a predefined metric or a customized metric. https://docs.aws.amazon.com/sdkforruby/api/Aws/AutoScaling/Types/PredefinedMetricSpecification.html", + "additionalProperties" => false, + "required" => ["predefined_metric_type"], + "properties" => { + "predefined_metric_type" => { + "type" => "string", + "enum" => ["ASGAverageCPUUtilization", "ASGAverageNetworkIn", "ASGAverageNetworkOut", "ALBRequestCountPerTarget"], + "default" => "ASGAverageCPUUtilization" + } + } + }, + "customized_metric_specification" => { + "type" => "object", + "description" => "A customized metric. You can specify either a predefined metric or a customized metric. https://docs.aws.amazon.com/sdkforruby/api/Aws/AutoScaling/Types/TargetTrackingConfiguration.html#customized_metric_specification-instance_method", + "additionalProperties" => false, + "required" => ["metric_name", "namespace", "statistic"], + "properties" => { + "metric_name" => { + "type" => "string", + "description" => "The name of the attribute to monitor eg. CPUUtilization." + }, + "namespace" => { + "type" => "string", + "description" => "The name of container 'metric_name' belongs to eg. 'AWS/ApplicationELB'" + }, + "statistic" => { + "type" => "string", + "enum" => ["Average", "Minimum", "Maximum", "SampleCount", "Sum"] + }, + "unit" => { + "type" => "string", + "description" => "Associated with the 'metric', usually something like Megabits or Seconds" + }, + "dimensions" => { + "type" => "array", + "description" => "What to monitor XXX dig up an example", + "items" => { + "type" => "object", + "additionalProperties" => false, + "required" => ["name", "value"], + "properties" => { + "name" => { + "type" => "string", + "description" => "XXX dig up an example" + }, + "value" => { + "type" => "string", + "description" => "XXX dig up an example" + } + } + } + } + } + } + } + } + } + } + }, "ingress_rules" => { "items" => { "properties" => { @@ -592,11 +773,29 @@ def self.validateConfig(pool, configurator) MU.log "You must specify 'cooldown' and 'adjustment' when 'policy_type' is set to 'SimpleScaling'", MU::ERR ok = false end + unless policy['type'] + MU.log "You must specify a 'type' when 'policy_type' is set to 'SimpleScaling'", MU::ERR + ok = false + end + elsif policy["policy_type"] == "TargetTrackingScaling" + unless policy["target_tracking_configuration"] + MU.log "You must specify 'target_tracking_configuration' when 'policy_type' is set to 'TargetTrackingScaling'", MU::ERR + ok = false + end + unless policy["target_tracking_configuration"]["customized_metric_specification"] or + policy["target_tracking_configuration"]["predefined_metric_specification"] + MU.log "Your target_tracking_configuration must specify one of customized_metric_specification or predefined_metric_specification when 'policy_type' is set to 'TargetTrackingScaling'", MU::ERR + ok = false + end elsif policy["policy_type"] == "StepScaling" if policy["step_adjustments"].nil? || policy["step_adjustments"].empty? MU.log "You must specify 'step_adjustments' when 'policy_type' is set to 'StepScaling'", MU::ERR ok = false end + unless policy['type'] + MU.log "You must specify a 'type' when 'policy_type' is set to 'StepScaling'", MU::ERR + ok = false + end policy["step_adjustments"].each{ |step| if step["adjustment"].nil? diff --git a/modules/mu/config/alarm.rb b/modules/mu/config/alarm.rb index c037b7216..7062a4d2d 100644 --- a/modules/mu/config/alarm.rb +++ b/modules/mu/config/alarm.rb @@ -65,7 +65,6 @@ def self.common_properties "dimensions" => { "type" => "array", "description" => "What to monitor", -# "minItems" => 1, "items" => { "type" => "object", "additionalProperties" => false, diff --git a/modules/mu/config/server_pool.rb b/modules/mu/config/server_pool.rb index 7667c08b8..96e261a88 100644 --- a/modules/mu/config/server_pool.rb +++ b/modules/mu/config/server_pool.rb @@ -69,81 +69,6 @@ def self.schema If you specify subnets and Availability Zones with this call, ensure that the subnets' Availability Zones match the Availability Zones specified." }, - "scaling_policies" => { - "type" => "array", - "minItems" => 1, - "items" => { - "type" => "object", - "required" => ["name", "type"], - "additionalProperties" => false, - "description" => "A custom AWS Autoscale scaling policy for this pool.", - "properties" => { - "name" => { - "type" => "string" - }, - "alarms" => MU::Config::Alarm.inline, - "type" => { - "type" => "string", - "enum" => ["ChangeInCapacity", "ExactCapacity", "PercentChangeInCapacity"], - "description" => "Specifies whether 'adjustment' is an absolute number or a percentage of the current capacity. Valid values are ChangeInCapacity, ExactCapacity, and PercentChangeInCapacity." - }, - "adjustment" => { - "type" => "integer", - "description" => "The number of instances by which to scale. 'type' determines the interpretation of this number (e.g., as an absolute number or as a percentage of the existing Auto Scaling group size). A positive increment adds to the current capacity and a negative value removes from the current capacity. Used only when policy_type is set to 'SimpleScaling'" - }, - "cooldown" => { - "type" => "integer", - "default" => 1, - "description" => "The amount of time, in seconds, after a scaling activity completes and before the next scaling activity can start." - }, - "min_adjustment_magnitude" => { - "type" => "integer", - "description" => "Used when 'type' is set to 'PercentChangeInCapacity', the scaling policy changes the DesiredCapacity of the Auto Scaling group by at least the number of instances specified in the value." - }, - "policy_type" => { - "type" => "string", - "enum" => ["SimpleScaling", "StepScaling"], - "description" => "'StepScaling' will add capacity based on the magnitude of the alarm breach, 'SimpleScaling' will add capacity based on the 'adjustment' value provided. Defaults to 'SimpleScaling'.", - "default" => "SimpleScaling" - }, - "metric_aggregation_type" => { - "type" => "string", - "enum" => ["Minimum", "Maximum", "Average"], - "description" => "Defaults to 'Average' if not specified. Required when policy_type is set to 'StepScaling'", - "default" => "Average" - }, - "step_adjustments" => { - "type" => "array", - "minItems" => 1, - "items" => { - "type" => "object", - "title" => "admin", - "description" => "Requires policy_type 'StepScaling'", - "required" => ["adjustment"], - "additionalProperties" => false, - "properties" => { - "adjustment" => { - "type" => "integer", - "description" => "The number of instances by which to scale at this specific step. Postive value when adding capacity, negative value when removing capacity" - }, - "lower_bound" => { - "type" => "integer", - "description" => "The lower bound value in percentage points above/below the alarm threshold at which to add/remove capacity for this step. Positive value when adding capacity and negative when removing capacity. If this is the first step and capacity is being added this value will most likely be 0" - }, - "upper_bound" => { - "type" => "integer", - "description" => "The upper bound value in percentage points above/below the alarm threshold at which to add/remove capacity for this step. Positive value when adding capacity and negative when removing capacity. If this is the first step and capacity is being removed this value will most likely be 0" - } - } - } - }, - "estimated_instance_warmup" => { - "type" => "integer", - "description" => "Required when policy_type is set to 'StepScaling'" - } - } - } - }, #XXX this needs its own primitive and discovery mechanism "zones" => { "type" => "array", From 533e0335cb0708d7a0bb2f9cbb0b1b2bb7545e2d Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 19 Nov 2018 15:40:36 +0000 Subject: [PATCH 05/12] properly guard changes to Autoscale scheduled actions and scaling policies --- modules/mu.rb | 46 +++++++++++++++++ modules/mu/clouds/aws/server_pool.rb | 74 ++++++++++++++++++++-------- 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/modules/mu.rb b/modules/mu.rb index d69454c62..c8404298b 100644 --- a/modules/mu.rb +++ b/modules/mu.rb @@ -576,6 +576,52 @@ def self.mySSLDir; @@mySSLDir end + # Recursively compare two hashes. Intended to see when cloud API descriptions + # of existing resources differ from proposed changes so we know when to + # bother updating. + # @param hash1 [Hash]: The first hash + # @param hash2 [Hash]: The second hash + # @param missing_is_default [Boolean]: Assume that any element missing from hash2 but present in hash1 is a default value to be ignored + # @return [Boolean] + def self.hashCmp(hash1, hash2, missing_is_default: false) + return false if hash1.nil? + hash2.each_pair { |k, v| + if hash1[k].nil? + return false + end + } + if !missing_is_default + hash1.each_pair { |k, v| + if hash2[k].nil? + return false + end + } + end + + hash1.each_pair { |k, v| + if hash1[k].is_a?(Array) + return false if !missing_is_default and hash2[k].nil? + if !hash2[k].nil? + hash2[k].each { |item| + if !hash1[k].include?(item) + return false + end + } + end + elsif hash1[k].is_a?(Hash) and !hash2[k].nil? + result = hashCmp(hash1[k], hash2[k]) + return false if !result + else + if missing_is_default + return false if !hash2[k].nil? and hash1[k] != hash2[k] + else + return false if hash1[k] != hash2[k] + end + end + } + true + end + # Recursively turn a Ruby OpenStruct into a Hash # @param struct [OpenStruct] # @return [Hash] diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index b0125e21d..1babf566c 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -213,18 +213,11 @@ def listNodes # Called automatically by {MU::Deploy#createResources} def groom if @config['schedule'] - resp = MU::Cloud::AWS.autoscale(@config['region']).describe_scheduled_actions( + ext_actions = MU::Cloud::AWS.autoscale(@config['region']).describe_scheduled_actions( auto_scaling_group_name: @mu_name - ) - if resp and resp.scheduled_update_group_actions - resp.scheduled_update_group_actions.each { |s| - MU.log "Removing scheduled action #{s.scheduled_action_name} from AutoScale group #{@mu_name}" - MU::Cloud::AWS.autoscale(@config['region']).delete_scheduled_action( - auto_scaling_group_name: @mu_name, - scheduled_action_name: s.scheduled_action_name - ) - } - end + ).scheduled_update_group_actions + + @config['schedule'].each { |s| sched_config = { :auto_scaling_group_name => @mu_name, @@ -236,10 +229,27 @@ def groom ['start_time', 'end_time'].each { |flag| sched_config[flag.to_sym] = Time.parse(s[flag]) if s[flag] } - MU.log "Adding scheduled action to AutoScale group #{@mu_name}", MU::NOTICE, details: sched_config - MU::Cloud::AWS.autoscale(@config['region']).put_scheduled_update_group_action( - sched_config - ) + action_already_correct = false + ext_actions.each { |ext| + if s['action_name'] == ext.scheduled_action_name + if !MU.hashCmp(MU.structToHash(ext), sched_config, missing_is_default: true) + MU.log "Removing scheduled action #{s['action_name']} from AutoScale group #{@mu_name}" + MU::Cloud::AWS.autoscale(@config['region']).delete_scheduled_action( + auto_scaling_group_name: @mu_name, + scheduled_action_name: s['action_name'] + ) + else + action_already_correct = true + end + break + end + } + if !action_already_correct + MU.log "Adding scheduled action to AutoScale group #{@mu_name}", MU::NOTICE, details: sched_config + MU::Cloud::AWS.autoscale(@config['region']).put_scheduled_update_group_action( + sched_config + ) + end } end @@ -280,14 +290,14 @@ def groom asg_options[:max_size] = @config["max_size"] asg_options[:new_instances_protected_from_scale_in] = (@config['scale_in_protection'] == "all") tg_arns = [] - if asg_options[:target_group_arns] + if asg_options[:target_group_arns] MU::Cloud::AWS.autoscale(@config['region']).attach_load_balancer_target_groups( auto_scaling_group_name: @mu_name, target_group_arns: asg_options[:target_group_arns] ) tg_arns = asg_options[:target_group_arns].dup asg_options.delete(:target_group_arns) - end + end MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(asg_options) @@ -304,10 +314,15 @@ def groom end if @config["scaling_policies"] and @config["scaling_policies"].size > 0 + ext_pols = MU::Cloud::AWS.autoscale(@config['region']).describe_policies( + auto_scaling_group_name: @mu_name + ).scaling_policies + @config["scaling_policies"].each { |policy| + policy_name = @deploy.getResourceName("#{@config['name']}-#{policy['name']}") policy_params = { :auto_scaling_group_name => @mu_name, - :policy_name => @deploy.getResourceName("#{@config['name']}-#{policy['name']}"), + :policy_name => policy_name, :policy_type => policy['policy_type'] } @@ -355,9 +370,26 @@ def strToSym(hash) end policy_params[:min_adjustment_magnitude] = policy['min_adjustment_magnitude'] if !policy['min_adjustment_magnitude'].nil? - pp policy_params - # XXX remove before put - resp = MU::Cloud::AWS.autoscale(@config['region']).put_scaling_policy(policy_params) + + policy_already_correct = false + ext_pols.each { |ext| + if ext.policy_name == policy_name + if !MU.hashCmp(MU.structToHash(ext), policy_params, missing_is_default: true) + MU::Cloud::AWS.autoscale(@config['region']).delete_policy( + auto_scaling_group_name: @mu_name, + policy_name: policy_name + ) + else + policy_already_correct = true + end + break + end + } + if !policy_already_correct + MU.log "Putting scaling policy #{policy_name} for #{@mu_name}", MU::NOTICE, details: policy_params + resp = MU::Cloud::AWS.autoscale(@config['region']).put_scaling_policy(policy_params) + end + # If we are creating alarms for scaling policies we need to have the autoscaling policy ARN # To make life easier we're creating the alarms here From 7ba1ff0e4334b919a7800be9ca9ee05580af99da Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 19 Nov 2018 16:44:12 +0000 Subject: [PATCH 06/12] Shortcuts for common scaling policies: scale_with_alb_traffic scale_with_cpu scale_with_network_in scale_with_network_out --- modules/mu.rb | 2 +- modules/mu/clouds/aws/server_pool.rb | 84 ++++++++++++++++++++++------ 2 files changed, 69 insertions(+), 17 deletions(-) diff --git a/modules/mu.rb b/modules/mu.rb index c8404298b..ab21fbde0 100644 --- a/modules/mu.rb +++ b/modules/mu.rb @@ -609,7 +609,7 @@ def self.hashCmp(hash1, hash2, missing_is_default: false) } end elsif hash1[k].is_a?(Hash) and !hash2[k].nil? - result = hashCmp(hash1[k], hash2[k]) + result = hashCmp(hash1[k], hash2[k], missing_is_default: missing_is_default) return false if !result else if missing_is_default diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index 1babf566c..0db1cba98 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -313,10 +313,24 @@ def groom setScaleInProtection(0) end + ext_pols = MU::Cloud::AWS.autoscale(@config['region']).describe_policies( + auto_scaling_group_name: @mu_name + ).scaling_policies if @config["scaling_policies"] and @config["scaling_policies"].size > 0 - ext_pols = MU::Cloud::AWS.autoscale(@config['region']).describe_policies( - auto_scaling_group_name: @mu_name - ).scaling_policies + legit_policies = [] + @config["scaling_policies"].each { |policy| + legit_policies << @deploy.getResourceName("#{@config['name']}-#{policy['name']}") + } + # Delete any scaling policies we're not configured for + ext_pols.each { |ext| + if !legit_policies.include?(ext.policy_name) + MU.log "Scaling policy #{ext.policy_name} is not named in scaling_policies, removing from #{@mu_name}", MU::NOTICE + MU::Cloud::AWS.autoscale(@config['region']).delete_policy( + auto_scaling_group_name: @mu_name, + policy_name: ext.policy_name + ) + end + } @config["scaling_policies"].each { |policy| policy_name = @deploy.getResourceName("#{@config['name']}-#{policy['name']}") @@ -533,6 +547,22 @@ def self.schema(config) "description" => "Protect instances from scale-in termination. Can be 'all', 'initial' (essentially 'min_size'), or an number; note the number needs to be a string, so put it in quotes", "pattern" => "^(all|initial|\\d+)$" }, + "scale_with_alb_traffic" => { + "type" => "float", + "description" => "Shorthand for creating a target_tracking_configuration to scale on ALBRequestCountPerTarget with some reasonable defaults" + }, + "scale_with_cpu" => { + "type" => "float", + "description" => "Shorthand for creating a target_tracking_configuration to scale on ASGAverageCPUUtilization with some reasonable defaults" + }, + "scale_with_network_in" => { + "type" => "float", + "description" => "Shorthand for creating a target_tracking_configuration to scale on ASGAverageNetworkIn with some reasonable defaults" + }, + "scale_with_network_out" => { + "type" => "float", + "description" => "Shorthand for creating a target_tracking_configuration to scale on ASGAverageNetworkOut with some reasonable defaults" + }, "termination_policies" => { "type" => "array", "minItems" => 1, @@ -630,17 +660,10 @@ def self.schema(config) "default" => false }, "predefined_metric_specification" => { - "type" => "object", "description" => "A predefined metric. You can specify either a predefined metric or a customized metric. https://docs.aws.amazon.com/sdkforruby/api/Aws/AutoScaling/Types/PredefinedMetricSpecification.html", - "additionalProperties" => false, - "required" => ["predefined_metric_type"], - "properties" => { - "predefined_metric_type" => { - "type" => "string", - "enum" => ["ASGAverageCPUUtilization", "ASGAverageNetworkIn", "ASGAverageNetworkOut", "ALBRequestCountPerTarget"], - "default" => "ASGAverageCPUUtilization" - } - } + "type" => "string", + "enum" => ["ASGAverageCPUUtilization", "ASGAverageNetworkIn", "ASGAverageNetworkOut", "ALBRequestCountPerTarget"], + "default" => "ASGAverageCPUUtilization" }, "customized_metric_specification" => { "type" => "object", @@ -666,7 +689,7 @@ def self.schema(config) }, "dimensions" => { "type" => "array", - "description" => "What to monitor XXX dig up an example", + "description" => "What resource to monitor with the alarm we are implicitly declaring", "items" => { "type" => "object", "additionalProperties" => false, @@ -674,11 +697,11 @@ def self.schema(config) "properties" => { "name" => { "type" => "string", - "description" => "XXX dig up an example" + "description" => "The type of resource we're monitoring, e.g. InstanceId or AutoScalingGroupName" }, "value" => { "type" => "string", - "description" => "XXX dig up an example" + "description" => "The name or cloud identifier of the resource we're monitoring" } } } @@ -755,6 +778,29 @@ def self.validateConfig(pool, configurator) } end + scale_aliases = { + "scale_with_alb_traffic" => "ALBRequestCountPerTarget", + "scale_with_cpu" => "ASGAverageCPUUtilization", + "scale_with_network_in" => "ASGAverageNetworkIn", + "scale_with_network_out" => "ASGAverageNetworkOut" + } + + scale_aliases.keys.each { |sp| + if pool[sp] + pool['scaling_policies'] ||= [] + pool['scaling_policies'] << { + 'name' => 'alb', + 'adjustment' => 1, + 'policy_type' => "TargetTrackingScaling", + 'estimated_instance_warmup' => 60, + 'target_tracking_configuration' => { + 'target_value' => pool[sp], + 'predefined_metric_specification' => scale_aliases[sp] + } + } + end + } + if !pool["basis"]["launch_config"].nil? launch = pool["basis"]["launch_config"] launch['iam_policies'] ||= pool['iam_policies'] @@ -819,6 +865,12 @@ def self.validateConfig(pool, configurator) MU.log "Your target_tracking_configuration must specify one of customized_metric_specification or predefined_metric_specification when 'policy_type' is set to 'TargetTrackingScaling'", MU::ERR ok = false end + # we gloss over an annoying layer of indirection in the API here + if policy["target_tracking_configuration"]["predefined_metric_specification"] + policy["target_tracking_configuration"]["predefined_metric_specification"] = { + "predefined_metric_type" => policy["target_tracking_configuration"]["predefined_metric_specification"] + } + end elsif policy["policy_type"] == "StepScaling" if policy["step_adjustments"].nil? || policy["step_adjustments"].empty? MU.log "You must specify 'step_adjustments' when 'policy_type' is set to 'StepScaling'", MU::ERR From 9a70781b15b718e967488b444910624eed0231b9 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 19 Nov 2018 17:38:22 +0000 Subject: [PATCH 07/12] add missing example server_pool.yml --- modules/mu/clouds/aws/server_pool.rb | 4 +- modules/mu/config/server_pool.yml | 71 ++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 modules/mu/config/server_pool.yml diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index 0db1cba98..b6cd3ff0b 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -324,7 +324,7 @@ def groom # Delete any scaling policies we're not configured for ext_pols.each { |ext| if !legit_policies.include?(ext.policy_name) - MU.log "Scaling policy #{ext.policy_name} is not named in scaling_policies, removing from #{@mu_name}", MU::NOTICE + MU.log "Scaling policy #{ext.policy_name} is not named in scaling_policies, removing from #{@mu_name}", MU::NOTICE, details: ext MU::Cloud::AWS.autoscale(@config['region']).delete_policy( auto_scaling_group_name: @mu_name, policy_name: ext.policy_name @@ -789,7 +789,7 @@ def self.validateConfig(pool, configurator) if pool[sp] pool['scaling_policies'] ||= [] pool['scaling_policies'] << { - 'name' => 'alb', + 'name' => scale_aliases[sp], 'adjustment' => 1, 'policy_type' => "TargetTrackingScaling", 'estimated_instance_warmup' => 60, diff --git a/modules/mu/config/server_pool.yml b/modules/mu/config/server_pool.yml new file mode 100644 index 000000000..9af24d0e4 --- /dev/null +++ b/modules/mu/config/server_pool.yml @@ -0,0 +1,71 @@ +<% if $complexity == 'complex' %> +name: <%= server_pools_name %> +cloud: AWS +alarms: +- comparison_operator: "GreaterThanThreshold" + metric_name: "HTTPCode_Target_5XX_Count" + evaluation_periods: 1 + name: "HTTP_Target_500_Count" + period: 60 + statistic: "Sum" + threshold: 10 + enable_notifications: true + notification_endpoint: 'admin@example.com' + namespace: "AWS/ApplicationELB" +scale_in_protection: "initial" +scale_with_cpu: 50.0 +scaling_policies: +- name: simplediskreads + policy_type: SimpleScaling + adjustment: 1 + cooldown: 60 + type: ChangeInCapacity + alarms: + - name: simplenetwork + namespace: "AWS/EC2" + metric_name: DiskReadBytes + period: 60 + evaluation_periods: 1 + comparison_operator: GreaterThanThreshold + statistic: Sum + threshold: 1073741824 +- name: trackingnetworkin + policy_type: TargetTrackingScaling + estimated_instance_warmup: 600 + target_tracking_configuration: + target_value: 250.0 + predefined_metric_specification: ASGAverageNetworkIn +termination_policies: +- "OldestInstance" +wait_for_nodes: 1 +min_size: 2 +max_size: 2 +schedule: +- action_name: scale-down-over-night + recurrence: "30 6 * * *" + min_size: 1 + max_size: 1 +- action_name: scale-up-during-the-day + recurrence: "30 12 * * *" + min_size: 2 + max_size: 2 +ingress_rules: +- port: 80 + hosts: + - 0.0.0.0/0 +- port: 443 + hosts: + - 0.0.0.0/0 +basis: + launch_config: + name: <%= server_pools_name %> + size: t2.medium +<% else %> +name: <%= server_pools_name %> +min_size: 1 +max_size: 1 +basis: + launch_config: + name: <%= server_pools_name %> + size: t2.small +<% end %> From c8c13d8fd44b8c39ac40471b2377fd0af11c2b47 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 19 Nov 2018 18:44:46 +0000 Subject: [PATCH 08/12] make sure server_pool sample gets called by parser testing --- modules/tests/super_complex_bok.yml | 5 +++++ modules/tests/super_simple_bok.yml | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/tests/super_complex_bok.yml b/modules/tests/super_complex_bok.yml index de3c42c92..d02204bd8 100755 --- a/modules/tests/super_complex_bok.yml +++ b/modules/tests/super_complex_bok.yml @@ -15,6 +15,8 @@ parameters: default: superBoK_logs - name: queues_name default: superBoK_queues + - name: server_pools_name + default: superBoK_ServerPool <% $complexity = 'complex' %> @@ -27,6 +29,9 @@ logs: servers: - <%= include("../mu/config/server.yml") %> +server_pools: +- + <%= include("../mu/config/server_pool.yml") %> search_domains: - <%= include("../mu/config/search_domain.yml") %> diff --git a/modules/tests/super_simple_bok.yml b/modules/tests/super_simple_bok.yml index de3c42c92..e30d473cf 100755 --- a/modules/tests/super_simple_bok.yml +++ b/modules/tests/super_simple_bok.yml @@ -15,6 +15,8 @@ parameters: default: superBoK_logs - name: queues_name default: superBoK_queues + - name: server_pools_name + default: superBoK_ServerPool <% $complexity = 'complex' %> @@ -27,10 +29,12 @@ logs: servers: - <%= include("../mu/config/server.yml") %> +server_pools: +- + <%= include("../mu/config/server_pool.yml") %> search_domains: - <%= include("../mu/config/search_domain.yml") %> - databases: - <%= include("../mu/config/database.yml") %> From 5007795b377156801f182c7ef8ddaff6ef888798 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Sun, 27 Jan 2019 01:54:40 +0000 Subject: [PATCH 09/12] breakfix: region resolution in adminFirewallRuleset --- modules/mu/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mu/config.rb b/modules/mu/config.rb index 9f10cf7ed..3b692c1f3 100644 --- a/modules/mu/config.rb +++ b/modules/mu/config.rb @@ -1154,7 +1154,7 @@ def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil) acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true} acl.delete("vpc") if !acl["vpc"] - acl["region"] == region if !region.nil? and !region.empty? + acl["region"] = region if !region.nil? and !region.empty? @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl) return {"type" => "firewall_rule", "name" => name} end From c74efeaa029e166916bb26256446cabd259c909e Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Sun, 27 Jan 2019 18:37:49 +0000 Subject: [PATCH 10/12] AWS::Server interrupt-cleanup should now target region correctly --- modules/mu/clouds/aws/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mu/clouds/aws/server.rb b/modules/mu/clouds/aws/server.rb index c4b1d985a..9b9b33541 100644 --- a/modules/mu/clouds/aws/server.rb +++ b/modules/mu/clouds/aws/server.rb @@ -259,7 +259,7 @@ def create Thread.new { MU.dupGlobals(parent_thread_id) MU::Cloud::AWS::Server.removeIAMProfile(@mu_name) - MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, skipsnapshots: true) + MU::Cloud::AWS::Server.cleanup(noop: false, ignoremaster: false, skipsnapshots: true, region: @config['region']) } end end From d1279049d742b2298af82abbce642c01d009569f Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Fri, 15 Feb 2019 18:05:04 +0000 Subject: [PATCH 11/12] take a preferred_target_group arg when setting up TargetTrackingScaling --- modules/mu/clouds/aws/server_pool.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index b6cd3ff0b..9840a2e63 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -357,6 +357,7 @@ def strToSym(hash) newhash end policy_params[:target_tracking_configuration] = strToSym(policy['target_tracking_configuration']) + policy_params[:target_tracking_configuration].delete(:preferred_target_group) if policy_params[:target_tracking_configuration][:predefined_metric_specification] and policy_params[:target_tracking_configuration][:predefined_metric_specification][:predefined_metric_type] == "ALBRequestCountPerTarget" lb_path = nil @@ -364,10 +365,18 @@ def strToSym(hash) if @deploy.deployment["loadbalancers"].size > 1 MU.log "Multiple load balancers attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN end - if lb["targetgroups"].size > 1 - MU.log "Multiple target groups attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN + lb_path = if lb["targetgroups"].size > 1 + if policy['target_tracking_configuration']["preferred_target_group"] and + lb["targetgroups"][policy['target_tracking_configuration']["preferred_target_group"]] + lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"][policy['target_tracking_configuration']["preferred_target_group"]].split(/:/)[5] + else + if policy['target_tracking_configuration']["preferred_target_group"] + MU.log "preferred_target_group was set to '#{policy["preferred_target_group"]}' but I don't see a target group by that name", MU::WARN + end + MU.log "Multiple target groups attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN, details: lb["targetgroups"].keys + lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"].values.first.split(/:/)[5] + end end - lb_path = lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"].values.first.split(/:/)[5] policy_params[:target_tracking_configuration][:predefined_metric_specification][:resource_label] = lb_path end @@ -654,6 +663,10 @@ def self.schema(config) "type" => "float", "description" => "The target value for the metric." }, + "preferred_target_group" => { + "type" => "string", + "description" => "If our load balancer has multiple target groups, prefer the one with this name instead of choosing one arbitrarily" + }, "disable_scale_in" => { "type" => "boolean", "description" => "If set to true, new instances created by this policy will not be subject to termination by scaling in.", From 964c945631a310bc57b157534f61aa4987eec60b Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Tue, 19 Feb 2019 22:18:04 +0000 Subject: [PATCH 12/12] expose AutoScale's simple notification support --- modules/mu/clouds/aws/server_pool.rb | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/modules/mu/clouds/aws/server_pool.rb b/modules/mu/clouds/aws/server_pool.rb index 9840a2e63..d34deddd1 100644 --- a/modules/mu/clouds/aws/server_pool.rb +++ b/modules/mu/clouds/aws/server_pool.rb @@ -212,6 +212,28 @@ def listNodes # Called automatically by {MU::Deploy#createResources} def groom + if @config['notifications'] and @config['notifications']['topic'] + arn = if @config['notifications']['topic'].match(/^arn:/) + @config['notifications']['topic'] + else + "arn:#{MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws"}:sns:#{@config['region']}:#{MU.account_number}:#{@config['notifications']['topic']}" + end + eventmap = { + "launch" => "autoscaling:EC2_INSTANCE_LAUNCH", + "failed_launch" => "autoscaling:EC2_INSTANCE_LAUNCH_ERROR", + "terminate" => "autoscaling:EC2_INSTANCE_TERMINATE", + "failed_terminate" => "autoscaling:EC2_INSTANCE_TERMINATE_ERROR" + } + MU.log "Sending simple notifications (#{@config['notifications']['events'].join(", ")}) to #{arn}" + MU::Cloud::AWS.autoscale(@config['region']).put_notification_configuration( + auto_scaling_group_name: @mu_name, + topic_arn: arn, + notification_types: @config['notifications']['events'].map { |e| + eventmap[e] + } + ) + end + if @config['schedule'] ext_actions = MU::Cloud::AWS.autoscale(@config['region']).describe_scheduled_actions( auto_scaling_group_name: @mu_name @@ -495,6 +517,26 @@ def self.schema(config) toplevel_required = [] schema = { + "notifications" => { + "type" => "object", + "description" => "Send notifications to an SNS topic for basic AutoScaling events", + "properties" => { + "topic" => { + "type" => "string", + "description" => "The short name or ARN of an SNS topic which should receive notifications for basic Autoscaling events" + }, + "events" => { + "type" => "array", + "description" => "The AutoScaling events which should generate a notification", + "items" => { + "type" => "string", + "description" => "The AutoScaling events which should generate a notification", + "enum" => ["launch", "failed_launch", "terminate", "failed_terminate"] + }, + "default" => ["launch", "failed_launch", "terminate", "failed_terminate"] + } + } + }, "generate_iam_role" => { "type" => "boolean", "default" => true,