Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

target groups: preserve last used arns in case of ROLLBACK_IN_PROGRESS state #673

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion aws/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ type Adapter struct {
obsoleteInstances []string
stackTerminationProtection bool
stackTags map[string]string
stackLastTargerGroupARNs map[string][]string
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

stackLastTargerGroupARNs -> stackLastTargetGroupARNs

controllerID string
sslPolicy string
ipAddressType string
Expand Down Expand Up @@ -248,6 +249,7 @@ func NewAdapter(clusterID, newControllerID, vpcID string, debug, disableInstrume
nlbCrossZone: DefaultNLBCrossZone,
nlbHTTPEnabled: DefaultNLBHTTPEnabled,
customFilter: DefaultCustomFilter,
stackLastTargerGroupARNs: make(map[string][]string),
TargetCNI: &TargetCNIconfig{
Enabled: false,
TargetGroupCh: make(chan []string, 10),
Expand Down Expand Up @@ -626,13 +628,29 @@ func (a *Adapter) SecurityGroupID() string {
// FindManagedStacks returns all CloudFormation stacks containing the controller management tags
// that match the current cluster and are ready to be used. The stack status is used to filter.
func (a *Adapter) FindManagedStacks() ([]*Stack, error) {
stacks, err := findManagedStacks(a.cloudformation, a.ClusterID(), a.controllerID)
stacks, err := findManagedStacks(a.cloudformation, a.ClusterID(), a.controllerID, a.stackLastTargerGroupARNs)
if err != nil {
return nil, err
}
return stacks, nil
}

func (a *Adapter) UpdateStackLastTargetGroupARNs(stack *Stack) {
if _, ok := a.stackLastTargerGroupARNs[stack.Name]; !ok {
if len(stack.TargetGroupARNs) > 0 {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do you check this here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The check can be moved outside the function yes, I just thought it made sense here.

a.stackLastTargerGroupARNs[stack.Name] = stack.TargetGroupARNs
}
}
}

func (a *Adapter) GetStackLastTargetGroupARNs(stackName string) []string {
return a.stackLastTargerGroupARNs[stackName]
}

func (a *Adapter) CleanLastTargetGroupARNs() {
a.stackLastTargerGroupARNs = make(map[string][]string)
}

// UpdateTargetGroupsAndAutoScalingGroups updates Auto Scaling Groups
// config to have relevant Target Groups and registers/deregisters single
// instances (that do not belong to ASG) in relevant Target Groups.
Expand Down
14 changes: 12 additions & 2 deletions aws/cf.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudformation"
"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
log "github.com/sirupsen/logrus"
)

const (
Expand Down Expand Up @@ -500,13 +501,22 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack {
}
}

func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, controllerID string) ([]*Stack, error) {
func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, controllerID string, stacksLastTargetGroupARNs map[string][]string) ([]*Stack, error) {
stacks := make([]*Stack, 0)
err := svc.DescribeStacksPages(&cloudformation.DescribeStacksInput{},
func(page *cloudformation.DescribeStacksOutput, lastPage bool) bool {
for _, s := range page.Stacks {
if isManagedStack(s.Tags, clusterID, controllerID) {
stacks = append(stacks, mapToManagedStack(s))
stack := mapToManagedStack(s)
if len(stack.TargetGroupARNs) == 0 && stack.status == cloudformation.StackStatusRollbackInProgress {
Copy link
Member

@szuecs szuecs Jan 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why check len(stack.TargetGroupARNs) == 0?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm still thinking should it be for all states without output or only for rollback

if _, ok := stacksLastTargetGroupARNs[stack.Name]; ok {
log.Warnf("stack %s is in rolling back state, falling back to last saved output", stack.Name)
stack.TargetGroupARNs = stacksLastTargetGroupARNs[stack.Name]
} else {
log.Warnf("stack %s has no saved target groups, skipping", stack.Name)
Copy link
Member

@szuecs szuecs Jan 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how do you skip here? I don't see that you skip anything :)
Maybe use continue to not add it to "stacks", then you do not need the code in worker.go.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, thanks. Forgot the continue but I still need the code in worker.go that's where I reset and save "last" data

}
}
stacks = append(stacks, stack)
}
}
return true
Expand Down
23 changes: 19 additions & 4 deletions aws/cf_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,7 @@ func TestFindManagedStacks(t *testing.T) {
wantErr: false,
},
{
name: "successfull-call-with-rollback-status",
name: "successfull-call-with-one-rollback-status",
given: fake.CFOutputs{
DescribeStackPages: fake.R(nil, nil),
DescribeStacks: fake.R(&cloudformation.DescribeStacksOutput{
Expand All @@ -545,21 +545,36 @@ func TestFindManagedStacks(t *testing.T) {
},
Outputs: []*cloudformation.Output{},
},
{
StackName: aws.String("managed-stack"),
StackStatus: aws.String(cloudformation.StackStatusCreateComplete),
Tags: []*cloudformation.Tag{
cfTag(kubernetesCreatorTag, DefaultControllerID),
cfTag(clusterIDTagPrefix+"test-cluster", resourceLifecycleOwned),
cfTag(certificateARNTagPrefix+"cert-arn", time.Time{}.Format(time.RFC3339)),
},
Outputs: []*cloudformation.Output{
{OutputKey: aws.String(outputLoadBalancerDNSName), OutputValue: aws.String("example.com")},
{OutputKey: aws.String(outputTargetGroupARN), OutputValue: aws.String("tg-arn")},
},
},
},
}, nil),
},
want: []*Stack{
{
Name: "managed-stack-rolling-back",
Name: "managed-stack",
DNSName: "example.com",
CertificateARNs: map[string]time.Time{
"cert-arn": {},
},
TargetGroupARNs: []string{"tg-arn"},
tags: map[string]string{
kubernetesCreatorTag: DefaultControllerID,
clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned,
certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
},
status: cloudformation.StackStatusRollbackInProgress,
status: cloudformation.StackStatusCreateComplete,
HTTP2: true,
},
},
Expand Down Expand Up @@ -645,7 +660,7 @@ func TestFindManagedStacks(t *testing.T) {
} {
t.Run(ti.name, func(t *testing.T) {
c := &fake.CFClient{Outputs: ti.given}
got, err := findManagedStacks(c, "test-cluster", DefaultControllerID)
got, err := findManagedStacks(c, "test-cluster", DefaultControllerID, map[string][]string{})
if err != nil {
if !ti.wantErr {
t.Error("unexpected error", err)
Expand Down
5 changes: 5 additions & 0 deletions worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,15 @@ func doWork(
return problems.Add("failed to list managed stacks: %w", err)
}

awsAdapter.CleanLastTargetGroupARNs()

for _, stack := range stacks {
if err := stack.Err(); err != nil {
problems.Add("stack %s error: %w", stack.Name, err)
}
if len(stack.TargetGroupARNs) > 0 {
awsAdapter.UpdateStackLastTargetGroupARNs(stack)
}
}

err = awsAdapter.UpdateAutoScalingGroupsAndInstances()
Expand Down
Loading