From e343d92167004b5da51d269a8899e2ac13679d9e Mon Sep 17 00:00:00 2001 From: Mustafa Abdelrahman Date: Wed, 3 Jan 2024 15:06:04 +0100 Subject: [PATCH 1/4] Ingore stacks in `ROLLBACK_IN_PROGRESS` state Signed-off-by: Mustafa Abdelrahman --- aws/cf.go | 5 +++++ aws/cf_test.go | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/aws/cf.go b/aws/cf.go index c9ed72a7..46dc5c31 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -505,6 +505,11 @@ func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, con err := svc.DescribeStacksPages(&cloudformation.DescribeStacksInput{}, func(page *cloudformation.DescribeStacksOutput, lastPage bool) bool { for _, s := range page.Stacks { + + if s.StackStatus != nil && *s.StackStatus == cloudformation.StackStatusRollbackInProgress { + continue + } + if isManagedStack(s.Tags, clusterID, controllerID) { stacks = append(stacks, mapToManagedStack(s)) } diff --git a/aws/cf_test.go b/aws/cf_test.go index b0acad7e..8d79b99d 100644 --- a/aws/cf_test.go +++ b/aws/cf_test.go @@ -530,7 +530,7 @@ func TestFindManagedStacks(t *testing.T) { wantErr: false, }, { - name: "successfull-call-with-rollback-status", + name: "successfull-call-with-one-rollback-status", given: fake.CFOutputs{ DescribeStackPages: fake.R(nil, nil), DescribeStacks: fake.R(&cloudformation.DescribeStacksOutput{ @@ -545,21 +545,36 @@ func TestFindManagedStacks(t *testing.T) { }, Outputs: []*cloudformation.Output{}, }, + { + StackName: aws.String("managed-stack"), + StackStatus: aws.String(cloudformation.StackStatusCreateComplete), + Tags: []*cloudformation.Tag{ + cfTag(kubernetesCreatorTag, DefaultControllerID), + cfTag(clusterIDTagPrefix+"test-cluster", resourceLifecycleOwned), + cfTag(certificateARNTagPrefix+"cert-arn", time.Time{}.Format(time.RFC3339)), + }, + Outputs: []*cloudformation.Output{ + {OutputKey: aws.String(outputLoadBalancerDNSName), OutputValue: aws.String("example.com")}, + {OutputKey: aws.String(outputTargetGroupARN), OutputValue: aws.String("tg-arn")}, + }, + }, }, }, nil), }, want: []*Stack{ { - Name: "managed-stack-rolling-back", + Name: "managed-stack", + DNSName: "example.com", CertificateARNs: map[string]time.Time{ "cert-arn": {}, }, + TargetGroupARNs: []string{"tg-arn"}, tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), }, - status: cloudformation.StackStatusRollbackInProgress, + status: cloudformation.StackStatusCreateComplete, HTTP2: true, }, }, From 4341dc24ca002a6adbc0e9c8eb1442ef026ceb22 Mon Sep 17 00:00:00 2001 From: Mustafa Abdelrahman Date: Thu, 4 Jan 2024 16:38:52 +0100 Subject: [PATCH 2/4] Store last target group ARNs Signed-off-by: Mustafa Abdelrahman --- aws/adapter.go | 20 +++++++++++++++++++- aws/cf.go | 22 +++++++++++++++++++--- aws/cf_test.go | 2 +- worker.go | 5 +++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/aws/adapter.go b/aws/adapter.go index 19df8977..12c0b1cb 100644 --- a/aws/adapter.go +++ b/aws/adapter.go @@ -61,6 +61,7 @@ type Adapter struct { obsoleteInstances []string stackTerminationProtection bool stackTags map[string]string + stackLastTargerGroupARNs map[string][]string controllerID string sslPolicy string ipAddressType string @@ -248,6 +249,7 @@ func NewAdapter(clusterID, newControllerID, vpcID string, debug, disableInstrume nlbCrossZone: DefaultNLBCrossZone, nlbHTTPEnabled: DefaultNLBHTTPEnabled, customFilter: DefaultCustomFilter, + stackLastTargerGroupARNs: make(map[string][]string), TargetCNI: &TargetCNIconfig{ Enabled: false, TargetGroupCh: make(chan []string, 10), @@ -626,13 +628,29 @@ func (a *Adapter) SecurityGroupID() string { // FindManagedStacks returns all CloudFormation stacks containing the controller management tags // that match the current cluster and are ready to be used. The stack status is used to filter. func (a *Adapter) FindManagedStacks() ([]*Stack, error) { - stacks, err := findManagedStacks(a.cloudformation, a.ClusterID(), a.controllerID) + stacks, err := findManagedStacks(a.cloudformation, a.ClusterID(), a.controllerID, a.stackLastTargerGroupARNs) if err != nil { return nil, err } return stacks, nil } +func (a *Adapter) UpdateStackLastTargetGroupARNs(stack *Stack) { + if _, ok := a.stackLastTargerGroupARNs[stack.Name]; !ok { + if len(stack.TargetGroupARNs) > 0 { + a.stackLastTargerGroupARNs[stack.Name] = stack.TargetGroupARNs + } + } +} + +func (a *Adapter) GetStackLastTargetGroupARNs(stackName string) []string { + return a.stackLastTargerGroupARNs[stackName] +} + +func (a *Adapter) CleanLastTargetGroupARNs() { + a.stackLastTargerGroupARNs = make(map[string][]string) +} + // UpdateTargetGroupsAndAutoScalingGroups updates Auto Scaling Groups // config to have relevant Target Groups and registers/deregisters single // instances (that do not belong to ASG) in relevant Target Groups. diff --git a/aws/cf.go b/aws/cf.go index 46dc5c31..4c988c6d 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" + log "github.com/sirupsen/logrus" ) const ( @@ -480,10 +481,20 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack { http2 = false } + arns := outputs.targetGroupARNs() + + // if *stack.StackStatus == cloudformation.StackStatusRollbackInProgress { + log.Warnf("stack %s is in rollback state", *stack.StackName) + for _, output := range stack.Outputs { + fmt.Printf("Output Key: %s, Value: %s\n", aws.StringValue(output.OutputKey), aws.StringValue(output.OutputValue)) + } + // outputs = newStackOutput(stackOutputs.Stacks[0].Outputs) + // } + return &Stack{ Name: aws.StringValue(stack.StackName), DNSName: outputs.dnsName(), - TargetGroupARNs: outputs.targetGroupARNs(), + TargetGroupARNs: arns, Scheme: parameters[parameterLoadBalancerSchemeParameter], SecurityGroup: parameters[parameterLoadBalancerSecurityGroupParameter], SSLPolicy: parameters[parameterListenerSslPolicyParameter], @@ -500,7 +511,7 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack { } } -func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, controllerID string) ([]*Stack, error) { +func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, controllerID string, stacksLastTargetGroupARNs map[string][]string) ([]*Stack, error) { stacks := make([]*Stack, 0) err := svc.DescribeStacksPages(&cloudformation.DescribeStacksInput{}, func(page *cloudformation.DescribeStacksOutput, lastPage bool) bool { @@ -511,7 +522,12 @@ func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, con } if isManagedStack(s.Tags, clusterID, controllerID) { - stacks = append(stacks, mapToManagedStack(s)) + stack := mapToManagedStack(s) + if len(stack.TargetGroupARNs) == 0 && stack.status == cloudformation.StackStatusRollbackInProgress { + log.Warnf("stack %s has no target groups in , falling back to last saved output", stack.Name) + stack.TargetGroupARNs = stacksLastTargetGroupARNs[stack.Name] + } + stacks = append(stacks, stack) } } return true diff --git a/aws/cf_test.go b/aws/cf_test.go index 8d79b99d..43d743c5 100644 --- a/aws/cf_test.go +++ b/aws/cf_test.go @@ -660,7 +660,7 @@ func TestFindManagedStacks(t *testing.T) { } { t.Run(ti.name, func(t *testing.T) { c := &fake.CFClient{Outputs: ti.given} - got, err := findManagedStacks(c, "test-cluster", DefaultControllerID) + got, err := findManagedStacks(c, "test-cluster", DefaultControllerID, map[string][]string{}) if err != nil { if !ti.wantErr { t.Error("unexpected error", err) diff --git a/worker.go b/worker.go index 86e9beea..aa186e45 100644 --- a/worker.go +++ b/worker.go @@ -287,10 +287,15 @@ func doWork( return problems.Add("failed to list managed stacks: %w", err) } + awsAdapter.CleanLastTargetGroupARNs() + for _, stack := range stacks { if err := stack.Err(); err != nil { problems.Add("stack %s error: %w", stack.Name, err) } + if len(stack.TargetGroupARNs) > 0 { + awsAdapter.UpdateStackLastTargetGroupARNs(stack) + } } err = awsAdapter.UpdateAutoScalingGroupsAndInstances() From 7e570e4216966481a43606d11784f46b12c4bc3b Mon Sep 17 00:00:00 2001 From: Mustafa Abdelrahman Date: Thu, 4 Jan 2024 16:44:47 +0100 Subject: [PATCH 3/4] Refactor stack rollback handling and target group fallback --- aws/cf.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/aws/cf.go b/aws/cf.go index 4c988c6d..715b13be 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -481,20 +481,10 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack { http2 = false } - arns := outputs.targetGroupARNs() - - // if *stack.StackStatus == cloudformation.StackStatusRollbackInProgress { - log.Warnf("stack %s is in rollback state", *stack.StackName) - for _, output := range stack.Outputs { - fmt.Printf("Output Key: %s, Value: %s\n", aws.StringValue(output.OutputKey), aws.StringValue(output.OutputValue)) - } - // outputs = newStackOutput(stackOutputs.Stacks[0].Outputs) - // } - return &Stack{ Name: aws.StringValue(stack.StackName), DNSName: outputs.dnsName(), - TargetGroupARNs: arns, + TargetGroupARNs: outputs.targetGroupARNs(), Scheme: parameters[parameterLoadBalancerSchemeParameter], SecurityGroup: parameters[parameterLoadBalancerSecurityGroupParameter], SSLPolicy: parameters[parameterListenerSslPolicyParameter], @@ -524,8 +514,12 @@ func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, con if isManagedStack(s.Tags, clusterID, controllerID) { stack := mapToManagedStack(s) if len(stack.TargetGroupARNs) == 0 && stack.status == cloudformation.StackStatusRollbackInProgress { - log.Warnf("stack %s has no target groups in , falling back to last saved output", stack.Name) - stack.TargetGroupARNs = stacksLastTargetGroupARNs[stack.Name] + if _, ok := stacksLastTargetGroupARNs[stack.Name]; ok { + log.Warnf("stack %s is in rolling back state, falling back to last saved output", stack.Name) + stack.TargetGroupARNs = stacksLastTargetGroupARNs[stack.Name] + } else { + log.Warnf("stack %s has no saved target groups, skipping", stack.Name) + } } stacks = append(stacks, stack) } From b0a43013f702909777cf931f045dc73d215594d3 Mon Sep 17 00:00:00 2001 From: Mustafa Abdelrahman Date: Thu, 4 Jan 2024 16:45:19 +0100 Subject: [PATCH 4/4] Remove unnecessary code for handling rollback in findManagedStacks function --- aws/cf.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/aws/cf.go b/aws/cf.go index 715b13be..0b266805 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -506,11 +506,6 @@ func findManagedStacks(svc cloudformationiface.CloudFormationAPI, clusterID, con err := svc.DescribeStacksPages(&cloudformation.DescribeStacksInput{}, func(page *cloudformation.DescribeStacksOutput, lastPage bool) bool { for _, s := range page.Stacks { - - if s.StackStatus != nil && *s.StackStatus == cloudformation.StackStatusRollbackInProgress { - continue - } - if isManagedStack(s.Tags, clusterID, controllerID) { stack := mapToManagedStack(s) if len(stack.TargetGroupARNs) == 0 && stack.status == cloudformation.StackStatusRollbackInProgress {