320 automated ami upgrades + upgrade locking

.github/DEVELOPER.md

@@ -55,6 +55,13 @@ You can also run `make coverage` to generate a coverage report.
 
 ## Running BDD tests
 
+### Dependencies
+
+1. You will need an existing EKS cluster running with the connection details exported into a kube config file.
+2. [Keikoproj Minion-Manager](https://github.com/keikoproj/minion-manager) must also be running in the cluster
+3. Instance Manager needs to be started outside of the bdd test suite
+
+
 Export some variables and run `make bdd` to run a functional e2e test.
 
 ### Example
@@ -96,3 +103,5 @@ testing: warning: no tests to run
 PASS
 ok  	github.com/keikoproj/instance-manager/test-bdd	1362.336s [no tests to run]
 ```
+
+Note: If your test cluster uses `InstanceGroups` to run core components, annotating the namespace with `instancemgr.keikoproj.io/config-excluded="true"` can help prevent unexpected disruption.

api/v1alpha1/instancegroup_types.go

@@ -47,8 +47,9 @@ const (
 	ReconcileModified  ReconcileState = "ReconcileModified"
 
 	// End States
-	ReconcileReady ReconcileState = "Ready"
-	ReconcileErr   ReconcileState = "Error"
+	ReconcileLocked ReconcileState = "Locked"
+	ReconcileReady  ReconcileState = "Ready"
+	ReconcileErr    ReconcileState = "Error"
 
 	// Userdata bootstrap stages
 	PreBootstrapStage  = "PreBootstrap"
@@ -76,6 +77,8 @@ const (
 	HostPlacementTenancyType      = "host"
 	DefaultPlacementTenancyType   = "default"
 	DedicatedPlacementTenancyType = "dedicated"
+
+	ImageLatestValue = "latest"
 )
 
 type ContainerRuntime string
@@ -87,6 +90,8 @@ const (
 
 	DockerRuntime     ContainerRuntime = "dockerd"
 	ContainerDRuntime ContainerRuntime = "containerd"
+
+	UpgradeLockedAnnotationKey = "instancemgr.keikoproj.io/lock-upgrades"
 )
 
 var (
@@ -392,6 +397,15 @@ func (ig *InstanceGroup) GetUpgradeStrategy() *AwsUpgradeStrategy {
 func (ig *InstanceGroup) SetUpgradeStrategy(strategy AwsUpgradeStrategy) {
 	ig.Spec.AwsUpgradeStrategy = strategy
 }
+func (ig *InstanceGroup) Locked() bool {
+	annotations := ig.GetAnnotations()
+	if val, ok := annotations[UpgradeLockedAnnotationKey]; ok {
+		if strings.EqualFold(val, "true") {
+			return true
+		}
+	}
+	return false
+}
 
 func (s *EKSSpec) Validate() error {
 	var (
@@ -521,7 +535,6 @@ func (c *EKSConfiguration) Validate() error {
 		c.SuspendedProcesses = processes
 	}
 
-
 	if c.BootstrapOptions != nil {
 		if c.BootstrapOptions.ContainerRuntime != "" && !contains(AllowedContainerRuntimes, c.BootstrapOptions.ContainerRuntime) {
 			return errors.Errorf("validation failed, 'bootstrapOptions.containerRuntime' must be one of %+v", AllowedContainerRuntimes)

api/v1alpha1/instancegroup_types_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 type EksUnitTest struct {
@@ -117,13 +118,13 @@ func TestInstanceGroupSpecValidate(t *testing.T) {
 					MinSize: 1,
 					Type:    "LaunchTemplate",
 					EKSConfiguration: &EKSConfiguration{
-						BootstrapOptions: &BootstrapOptions{ContainerRuntime: "foo"},
-						EksClusterName:        "my-eks-cluster",
-						NodeSecurityGroups:    []string{"sg-123456789"},
-						Image:                 "ami-12345",
-						InstanceType:          "m5.large",
-						KeyPairName:           "thisShouldBeOptional",
-						Subnets:               []string{"subnet-1111111", "subnet-222222"},
+						BootstrapOptions:   &BootstrapOptions{ContainerRuntime: "foo"},
+						EksClusterName:     "my-eks-cluster",
+						NodeSecurityGroups: []string{"sg-123456789"},
+						Image:              "ami-12345",
+						InstanceType:       "m5.large",
+						KeyPairName:        "thisShouldBeOptional",
+						Subnets:            []string{"subnet-1111111", "subnet-222222"},
 					},
 				}, nil, nil),
 			},
@@ -353,6 +354,41 @@ func TestInstanceGroupSpecValidate(t *testing.T) {
 	}
 }
 
+func TestLockedAnnotation(t *testing.T) {
+	tests := []struct {
+		name       string
+		annotation string
+		expected   bool
+	}{
+		{
+			name:       "Locked",
+			annotation: "true",
+			expected:   true,
+		},
+		{
+			name:       "Unlocked",
+			annotation: "false",
+			expected:   false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testIg := &InstanceGroup{
+				ObjectMeta: v1.ObjectMeta{
+					Annotations: map[string]string{
+						UpgradeLockedAnnotationKey: test.annotation,
+					},
+				},
+			}
+			res := testIg.Locked()
+			if res != test.expected {
+				t.Errorf("%v: got %v, expected %v", test.name, res, test.expected)
+			}
+		})
+	}
+}
+
 func basicFargateSpec() *EKSFargateSpec {
 	return &EKSFargateSpec{
 		ClusterName:         "",

controllers/instancegroup_controller.go

@@ -246,7 +246,7 @@ func (r *InstanceGroupReconciler) IsNamespaceAnnotated(namespace, key, value str
 		}
 
 		annotations := unstructuredNamespace.GetAnnotations()
-		if kubeprovider.HasAnnotation(annotations, key, value) {
+		if kubeprovider.HasAnnotationWithValue(annotations, key, value) {
 			return true
 		}
 	}

controllers/interface.go

@@ -16,6 +16,7 @@ type CloudDeployer interface {
 	GetState() v1alpha.ReconcileState // Gets the current state type of the instance group
 	SetState(v1alpha.ReconcileState)  // Sets the current state of the instance group
 	IsReady() bool                    // Returns true if state is Ready
+	Locked() bool                     // Returns true if instanceGroup is locked
 }
 
 func HandleReconcileRequest(d CloudDeployer) error {
@@ -54,6 +55,11 @@ func HandleReconcileRequest(d CloudDeployer) error {
 
 	// CRUD Nodes Upgrade Strategy
 	if d.GetState() == v1alpha.ReconcileInitUpgrade {
+		// Locked
+		if d.Locked() {
+			d.SetState(v1alpha.ReconcileLocked)
+			return nil
+		}
 		err = d.UpgradeNodes()
 		if err != nil {
 			return err
@@ -67,12 +73,18 @@ func HandleReconcileRequest(d CloudDeployer) error {
 
 	// Bootstrap Nodes
 	if d.IsReady() {
+
 		err = d.BootstrapNodes()
 		if err != nil {
 			return err
 		}
 
 		if d.GetState() == v1alpha.ReconcileInitUpgrade {
+			// Locked
+			if d.Locked() {
+				d.SetState(v1alpha.ReconcileLocked)
+				return nil
+			}
 			err = d.UpgradeNodes()
 			if err != nil {
 				return err

controllers/providers/aws/aws.go

@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/eks/eksiface"
 	"github.com/aws/aws-sdk-go/service/iam/iamiface"
+	"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
 	"github.com/pkg/errors"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
@@ -55,6 +56,7 @@ const (
 	DescribeLaunchTemplateVersionsTTL time.Duration = 60 * time.Second
 	DescribeInstanceTypesTTL          time.Duration = 24 * time.Hour
 	DescribeInstanceTypeOfferingTTL   time.Duration = 1 * time.Hour
+	GetParameterTTL                   time.Duration = 1 * time.Hour
 
 	CacheBackgroundPruningInterval time.Duration = 1 * time.Hour
 	CacheMaxItems                  int64         = 250
@@ -117,6 +119,7 @@ type AwsWorker struct {
 	EksClient   eksiface.EKSAPI
 	IamClient   iamiface.IAMAPI
 	Ec2Client   ec2iface.EC2API
+	SsmClient   ssmiface.SSMAPI
 	Ec2Metadata *ec2metadata.EC2Metadata
 	Parameters  map[string]interface{}
 }
@@ -246,10 +249,9 @@ func GetScalingConfigName(group *autoscaling.Group) string {
 }
 
 func GetInstanceTypeNetworkInfo(instanceTypes []*ec2.InstanceTypeInfo, instanceType string) *ec2.NetworkInfo {
-	for _, instanceTypeInfo := range instanceTypes {
-		if aws.StringValue(instanceTypeInfo.InstanceType) == instanceType {
-			return instanceTypeInfo.NetworkInfo
-		}
+	i := GetInstanceTypeInfo(instanceTypes, instanceType)
+	if i != nil {
+		return i.NetworkInfo
 	}
 	return nil
 }
@@ -262,3 +264,11 @@ func GetInstanceTypeInfo(instanceTypes []*ec2.InstanceTypeInfo, instanceType str
 	}
 	return nil
 }
+
+func GetInstanceTypeArchitectures(instanceTypes []*ec2.InstanceTypeInfo, instanceType string) []string {
+	i := GetInstanceTypeInfo(instanceTypes, instanceType)
+	if i != nil {
+		return aws.StringValueSlice((*i).ProcessorInfo.SupportedArchitectures)
+	}
+	return nil
+}

controllers/providers/aws/ssm.go

@@ -0,0 +1,72 @@
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ssm"
+	"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
+	"github.com/keikoproj/aws-sdk-go-cache/cache"
+	"github.com/keikoproj/instance-manager/controllers/common"
+)
+
+type architectureMap map[string]string
+
+const (
+	EksOptimisedAmiPath           = "/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id"
+	EksOptimisedAmazonLinux2Arm64 = "/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id"
+	EksOptimisedBottlerocket      = "/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id"
+	EksOptimisedBottlerocketArm64 = "/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id"
+	EksOptimisedWindowsCore       = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id"
+	EksOptimisedWindowsFull       = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-%s/image_id"
+)
+
+var (
+	EksAmis = map[string]architectureMap{
+		"amazonlinux2": architectureMap{
+			"x86_64": EksOptimisedAmiPath,
+			"arm64":  EksOptimisedAmazonLinux2Arm64,
+		},
+		"bottlerocket": architectureMap{
+			"x86_64": EksOptimisedBottlerocket,
+			"arm64":  EksOptimisedBottlerocketArm64,
+		},
+		"windows": architectureMap{
+			"x86_64": EksOptimisedWindowsCore,
+		},
+	}
+)
+
+func GetAwsSsmClient(region string, cacheCfg *cache.Config, maxRetries int, collector *common.MetricsCollector) ssmiface.SSMAPI {
+	config := aws.NewConfig().WithRegion(region).WithCredentialsChainVerboseErrors(true)
+	config = request.WithRetryer(config, NewRetryLogger(maxRetries, collector))
+	sess, err := session.NewSession(config)
+	if err != nil {
+		panic(err)
+	}
+	cache.AddCaching(sess, cacheCfg)
+	cacheCfg.SetCacheTTL("ssm", "GetParameter", GetParameterTTL)
+	sess.Handlers.Complete.PushFront(func(r *request.Request) {
+		ctx := r.HTTPRequest.Context()
+		log.V(1).Info("AWS API call",
+			"cacheHit", cache.IsCacheHit(ctx),
+			"service", r.ClientInfo.ServiceName,
+			"operation", r.Operation.Name,
+		)
+	})
+	return ssm.New(sess)
+}
+
+func (w *AwsWorker) GetEksLatestAmi(OSFamily string, arch string, kubernetesVersion string) (string, error) {
+	input := &ssm.GetParameterInput{
+		Name: aws.String(fmt.Sprintf(EksAmis[OSFamily][arch], kubernetesVersion)),
+	}
+
+	output, err := w.SsmClient.GetParameter(input)
+	if err != nil {
+		return "", err
+	}
+	return aws.StringValue(output.Parameter.Value), nil
+}

controllers/providers/kubernetes/crd.go

@@ -329,7 +329,7 @@ func GetResources(kube dynamic.Interface, instanceGroup *v1alpha1.InstanceGroup,
 
 		annotations := ru.GetAnnotations()
 
-		if HasAnnotation(annotations, OwnershipAnnotationKey, OwnershipAnnotationValue) && HasAnnotation(annotations, ScopeAnnotationKey, status.GetActiveScalingGroupName()) {
+		if HasAnnotationWithValue(annotations, OwnershipAnnotationKey, OwnershipAnnotationValue) && HasAnnotationWithValue(annotations, ScopeAnnotationKey, status.GetActiveScalingGroupName()) {
 			if IsPathValue(ru, statusJSONPath, completedStatus) || IsPathValue(ru, statusJSONPath, errorStatus) {
 				// if resource is not completed and not failed, it must be still active
 				inactiveResources = append(inactiveResources, ru)

controllers/providers/kubernetes/utils.go

@@ -116,7 +116,14 @@ func AddAnnotation(u *unstructured.Unstructured, key, value string) {
 	u.SetAnnotations(annotations)
 }
 
-func HasAnnotation(annotations map[string]string, key, value string) bool {
+func HasAnnotation(annotations map[string]string, key string) bool {
+	if _, ok := annotations[key]; ok {
+		return true
+	}
+	return false
+}
+
+func HasAnnotationWithValue(annotations map[string]string, key, value string) bool {
 	if val, ok := annotations[key]; ok {
 		if strings.EqualFold(val, value) {
 			return true