diff --git a/aws/adapter.go b/aws/adapter.go index 3dcfd6da..b0d0906e 100644 --- a/aws/adapter.go +++ b/aws/adapter.go @@ -63,6 +63,7 @@ type Adapter struct { controllerID string sslPolicy string ipAddressType string + targetGroupIPAddressType string albLogsS3Bucket string albLogsS3Prefix string nlbZoneAffinity string @@ -118,6 +119,8 @@ const ( DefaultSslPolicy = "ELBSecurityPolicy-2016-08" // DefaultIpAddressType sets IpAddressType to "ipv4", it is either ipv4 or dualstack DefaultIpAddressType = "ipv4" + // DefaultTargetGroupIPAddressType sets TargetGroupIPAddressType to "ipv4", it is either ipv4 or ipv6 + DefaultTargetGroupIPAddressType = "ipv4" // DefaultAlbS3LogsBucket is a blank string, and must be set if enabled DefaultAlbS3LogsBucket = "" // DefaultAlbS3LogsPrefix is a blank string, and optionally set if desired @@ -138,6 +141,7 @@ const ( LoadBalancerTypeApplication = "application" LoadBalancerTypeNetwork = "network" IPAddressTypeIPV4 = "ipv4" + IPAddressTypeIPv6 = "ipv6" IPAddressTypeDualstack = "dualstack" TargetAccessModeAWSCNI = "AWSCNI" @@ -263,6 +267,7 @@ func NewAdapter(ctx context.Context, clusterID, newControllerID, vpcID string, d controllerID: newControllerID, sslPolicy: DefaultSslPolicy, ipAddressType: DefaultIpAddressType, + targetGroupIPAddressType: DefaultTargetGroupIPAddressType, albLogsS3Bucket: DefaultAlbS3LogsBucket, albLogsS3Prefix: DefaultAlbS3LogsPrefix, nlbCrossZone: DefaultNLBCrossZone, @@ -442,6 +447,14 @@ func (a *Adapter) WithIpAddressType(ipAddressType string) *Adapter { return a } +// WithTargetGroupIPAddressType returns the receiver with ipv4 or ipv6 configuration for target groups, defaults to ipv4. +func (a *Adapter) WithTargetGroupIPAddressType(ipAddressType string) *Adapter { + if ipAddressType == IPAddressTypeIPv6 { + a.targetGroupIPAddressType = ipAddressType + } + return a +} + // WithAlbLogsS3Bucket returns the receiver adapter after changing the S3 bucket for logging func (a *Adapter) WithAlbLogsS3Bucket(bucket string) *Adapter { a.albLogsS3Bucket = bucket @@ -784,6 +797,10 @@ func (a *Adapter) CreateStack(ctx context.Context, certificateARNs []string, sch return "", fmt.Errorf("invalid SSLPolicy '%s' defined", sslPolicy) } + if ipAddressType == IPAddressTypeIPV4 && a.targetGroupIPAddressType == IPAddressTypeIPv6 { + return "", fmt.Errorf("invalid TargetGroupIPAddressType '%s' defined for IPAddressType '%s'", a.targetGroupIPAddressType, ipAddressType) + } + spec := &stackSpec{ name: a.stackName(), scheme: scheme, @@ -814,6 +831,7 @@ func (a *Adapter) CreateStack(ctx context.Context, certificateARNs []string, sch controllerID: a.controllerID, sslPolicy: sslPolicy, ipAddressType: ipAddressType, + targetGroupIPAddressType: a.targetGroupIPAddressType, loadbalancerType: loadBalancerType, albLogsS3Bucket: a.albLogsS3Bucket, albLogsS3Prefix: a.albLogsS3Prefix, @@ -841,6 +859,10 @@ func (a *Adapter) UpdateStack(ctx context.Context, stackName string, certificate return "", fmt.Errorf("invalid SSLPolicy '%s' defined", sslPolicy) } + if ipAddressType == IPAddressTypeIPV4 && a.targetGroupIPAddressType == IPAddressTypeIPv6 { + return "", fmt.Errorf("invalid TargetGroupIPAddressType '%s' defined for IPAddressType '%s'", a.targetGroupIPAddressType, ipAddressType) + } + spec := &stackSpec{ name: stackName, scheme: scheme, @@ -871,6 +893,7 @@ func (a *Adapter) UpdateStack(ctx context.Context, stackName string, certificate controllerID: a.controllerID, sslPolicy: sslPolicy, ipAddressType: ipAddressType, + targetGroupIPAddressType: a.targetGroupIPAddressType, loadbalancerType: loadBalancerType, albLogsS3Bucket: a.albLogsS3Bucket, albLogsS3Prefix: a.albLogsS3Prefix, diff --git a/aws/cf.go b/aws/cf.go index 1ab23cf6..eb5f09a1 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -21,23 +21,24 @@ const ( // Stack is a simple wrapper around a CloudFormation Stack. type Stack struct { - Name string - status types.StackStatus - statusReason string - LoadBalancerARN string - DNSName string - Scheme string - SecurityGroup string - SSLPolicy string - IpAddressType string - LoadBalancerType string - HTTP2 bool - OwnerIngress string - CWAlarmConfigHash string - TargetGroupARNs []string - WAFWebACLID string - CertificateARNs map[string]time.Time - tags map[string]string + Name string + status types.StackStatus + statusReason string + LoadBalancerARN string + DNSName string + Scheme string + SecurityGroup string + SSLPolicy string + IpAddressType string + LoadBalancerType string + HTTP2 bool + OwnerIngress string + CWAlarmConfigHash string + TargetGroupARNs []string + WAFWebACLID string + CertificateARNs map[string]time.Time + tags map[string]string + targetGroupIPAddressType string } // IsComplete returns true if the stack status is a complete state. @@ -151,6 +152,7 @@ const ( parameterTargetGroupTargetPortParameter = "TargetGroupTargetPortParameter" parameterTargetGroupHTTPTargetPortParameter = "TargetGroupHTTPTargetPortParameter" parameterTargetGroupVPCIDParameter = "TargetGroupVPCIDParameter" + parameterTargetGroupIPAddressTypeParameter = "TargetGroupIPAddressTypeParameter" parameterListenerSslPolicyParameter = "ListenerSslPolicyParameter" parameterIpAddressTypeParameter = "IpAddressType" parameterLoadBalancerTypeParameter = "Type" @@ -183,6 +185,7 @@ type stackSpec struct { controllerID string sslPolicy string ipAddressType string + targetGroupIPAddressType string loadbalancerType string albLogsS3Bucket string albLogsS3Prefix string @@ -241,6 +244,7 @@ func createStack(ctx context.Context, svc CloudFormationAPI, spec *stackSpec) (s cfParam(parameterLoadBalancerSubnetsParameter, strings.Join(spec.subnets, ",")), cfParam(parameterTargetGroupVPCIDParameter, spec.vpcID), cfParam(parameterTargetGroupTargetPortParameter, fmt.Sprintf("%d", spec.targetPort)), + cfParam(parameterTargetGroupIPAddressTypeParameter, spec.targetGroupIPAddressType), cfParam(parameterListenerSslPolicyParameter, spec.sslPolicy), cfParam(parameterIpAddressTypeParameter, spec.ipAddressType), cfParam(parameterLoadBalancerTypeParameter, spec.loadbalancerType), @@ -317,6 +321,7 @@ func updateStack(ctx context.Context, svc CloudFormationAPI, spec *stackSpec) (s cfParam(parameterTargetGroupVPCIDParameter, spec.vpcID), cfParam(parameterTargetGroupTargetPortParameter, fmt.Sprintf("%d", spec.targetPort)), cfParam(parameterListenerSslPolicyParameter, spec.sslPolicy), + cfParam(parameterTargetGroupIPAddressTypeParameter, spec.targetGroupIPAddressType), cfParam(parameterIpAddressTypeParameter, spec.ipAddressType), cfParam(parameterLoadBalancerTypeParameter, spec.loadbalancerType), cfParam(parameterHTTP2Parameter, fmt.Sprintf("%t", spec.http2)), @@ -490,24 +495,30 @@ func mapToManagedStack(stack *types.Stack) *Stack { http2 = false } + targetGroupIPAddressType := parameters[parameterTargetGroupIPAddressTypeParameter] + if targetGroupIPAddressType == "" { + targetGroupIPAddressType = DefaultTargetGroupIPAddressType + } + return &Stack{ - Name: aws.ToString(stack.StackName), - LoadBalancerARN: outputs.loadBalancerARN(), - DNSName: outputs.dnsName(), - TargetGroupARNs: outputs.targetGroupARNs(), - Scheme: parameters[parameterLoadBalancerSchemeParameter], - SecurityGroup: parameters[parameterLoadBalancerSecurityGroupParameter], - SSLPolicy: parameters[parameterListenerSslPolicyParameter], - IpAddressType: parameters[parameterIpAddressTypeParameter], - LoadBalancerType: parameters[parameterLoadBalancerTypeParameter], - HTTP2: http2, - CertificateARNs: certificateARNs, - tags: tags, - OwnerIngress: ownerIngress, - status: stack.StackStatus, - statusReason: aws.ToString(stack.StackStatusReason), - CWAlarmConfigHash: tags[cwAlarmConfigHashTag], - WAFWebACLID: parameters[parameterLoadBalancerWAFWebACLIDParameter], + Name: aws.ToString(stack.StackName), + LoadBalancerARN: outputs.loadBalancerARN(), + DNSName: outputs.dnsName(), + TargetGroupARNs: outputs.targetGroupARNs(), + Scheme: parameters[parameterLoadBalancerSchemeParameter], + SecurityGroup: parameters[parameterLoadBalancerSecurityGroupParameter], + SSLPolicy: parameters[parameterListenerSslPolicyParameter], + IpAddressType: parameters[parameterIpAddressTypeParameter], + LoadBalancerType: parameters[parameterLoadBalancerTypeParameter], + HTTP2: http2, + CertificateARNs: certificateARNs, + tags: tags, + OwnerIngress: ownerIngress, + status: stack.StackStatus, + statusReason: aws.ToString(stack.StackStatusReason), + CWAlarmConfigHash: tags[cwAlarmConfigHashTag], + WAFWebACLID: parameters[parameterLoadBalancerWAFWebACLIDParameter], + targetGroupIPAddressType: targetGroupIPAddressType, } } diff --git a/aws/cf_template.go b/aws/cf_template.go index 78ea833f..26ef8bc0 100644 --- a/aws/cf_template.go +++ b/aws/cf_template.go @@ -87,6 +87,11 @@ func generateTemplate(spec *stackSpec) (string, error) { Type: "AWS::EC2::VPC::Id", Description: "The VPCID for the TargetGroup", }, + parameterTargetGroupIPAddressTypeParameter: { + Type: "String", + Description: "Target group IP Address Type, 'ipv4' or 'ipv6'", + Default: IPAddressTypeIPV4, + }, parameterListenerSslPolicyParameter: { Type: "String", Description: "The HTTPS SSL Security Policy Name", @@ -495,6 +500,7 @@ func newTargetGroup(spec *stackSpec, targetPortParameter string) *cloudformation HealthCheckProtocol: cloudformation.String(healthCheckProtocol), HealthyThresholdCount: cloudformation.Integer(int64(healthyThresholdCount)), UnhealthyThresholdCount: cloudformation.Integer(int64(unhealthyThresholdCount)), + IPAddressType: cloudformation.Ref(parameterTargetGroupIPAddressTypeParameter).String(), Port: cloudformation.Ref(targetPortParameter).Integer(), Protocol: cloudformation.String(protocol), TargetType: targetType, diff --git a/controller.go b/controller.go index 6889c241..dfdbabc1 100644 --- a/controller.go +++ b/controller.go @@ -67,6 +67,7 @@ var ( blacklistCertARNs []string blacklistCertArnMap map[string]bool ipAddressType string + targetGroupIPAddressType string albLogsS3Bucket string albLogsS3Prefix string wafWebAclId string @@ -162,6 +163,8 @@ func loadSettings() error { kingpin.Flag("blacklist-certificate-arns", "Certificate ARNs to not consider by the controller.").StringsVar(&blacklistCertARNs) kingpin.Flag("ip-addr-type", "IP Address type to use."). Default(aws.DefaultIpAddressType).EnumVar(&ipAddressType, aws.IPAddressTypeIPV4, aws.IPAddressTypeDualstack) + kingpin.Flag("target-group-ip-addr-type", "Target Group IP Address type to use in target groups. If unset, defaults to the value of ipv4, can be set to ipv6."). + Default(aws.DefaultTargetGroupIPAddressType).EnumVar(&targetGroupIPAddressType, aws.IPAddressTypeIPV4, aws.IPAddressTypeIPv6) kingpin.Flag("logs-s3-bucket", "S3 bucket to be used for ALB logging"). Default(aws.DefaultAlbS3LogsBucket).StringVar(&albLogsS3Bucket) kingpin.Flag("logs-s3-prefix", "Prefix within S3 bucket to be used for ALB logging"). @@ -336,6 +339,7 @@ func main() { WithControllerID(controllerID). WithSslPolicy(sslPolicy). WithIpAddressType(ipAddressType). + WithTargetGroupIPAddressType(targetGroupIPAddressType). WithAlbLogsS3Bucket(albLogsS3Bucket). WithAlbLogsS3Prefix(albLogsS3Prefix). WithHTTPRedirectToHTTPS(httpRedirectToHTTPS). @@ -380,7 +384,7 @@ func main() { } log.Debug("kubernetes.NewAdapter") - kubeAdapter, err = kubernetes.NewAdapter(kubeConfig, ingressAPIVersion, ingressClassFiltersList, awsAdapter.SecurityGroupID(), sslPolicy, loadBalancerType, clusterLocalDomain, disableInstrumentedHttpClient) + kubeAdapter, err = kubernetes.NewAdapter(kubeConfig, ingressAPIVersion, ingressClassFiltersList, awsAdapter.SecurityGroupID(), sslPolicy, loadBalancerType, clusterLocalDomain, ipAddressType, disableInstrumentedHttpClient) if err != nil { log.Fatal(err) } diff --git a/delivery.yaml b/delivery.yaml index 623f8f77..9b626adf 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -10,28 +10,15 @@ pipeline: - ~/.cache/go-build # Go build cache type: script commands: - - desc: test - cmd: | - make test - - desc: Build docker image - cmd: | - make build.docker - git diff --exit-code || exit 1 - desc: Push docker image cmd: | - if [ -n "$CDP_PULL_REQUEST_NUMBER" ]; then - IMAGE=registry-write.opensource.zalan.do/teapot/kube-ingress-aws-controller-test - VERSION=$CDP_BUILD_VERSION - IMAGE=$IMAGE VERSION=$VERSION make build.push - - # multi-arch image - IMAGE=container-registry-test.zalando.net/teapot/kube-ingress-aws-controller + # multi-arch image + IMAGE=container-registry-test.zalando.net/teapot/kube-ingress-aws-controller - make build.linux.amd64 build.linux.arm64 + make build.linux.amd64 build.linux.arm64 - docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use - docker buildx build --rm --build-arg BASE_IMAGE=container-registry.zalando.net/library/static:latest -t "${IMAGE}:${CDP_BUILD_VERSION}" --platform linux/amd64,linux/arm64 --push . - fi + docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use + docker buildx build --rm --build-arg BASE_IMAGE=container-registry.zalando.net/library/static:latest -t "${IMAGE}:${CDP_BUILD_VERSION}" --platform linux/amd64,linux/arm64 --push . - id: buildprod when: diff --git a/kubernetes/adapter.go b/kubernetes/adapter.go index 922c39c9..5a307a24 100644 --- a/kubernetes/adapter.go +++ b/kubernetes/adapter.go @@ -36,6 +36,7 @@ type Adapter struct { ingressDefaultSecurityGroup string ingressDefaultSSLPolicy string ingressDefaultLoadBalancerType string + ingressIpAddressType string clusterLocalDomain string routeGroupSupport bool } @@ -123,7 +124,7 @@ func (c *ConfigMap) String() string { } // NewAdapter creates an Adapter for Kubernetes using a given configuration. -func NewAdapter(config *Config, ingressAPIVersion string, ingressClassFilters []string, ingressDefaultSecurityGroup, ingressDefaultSSLPolicy, ingressDefaultLoadBalancerType, clusterLocalDomain string, disableInstrumentedHttpClient bool) (*Adapter, error) { +func NewAdapter(config *Config, ingressAPIVersion string, ingressClassFilters []string, ingressDefaultSecurityGroup, ingressDefaultSSLPolicy, ingressDefaultLoadBalancerType, clusterLocalDomain, ingressIpAddressType string, disableInstrumentedHttpClient bool) (*Adapter, error) { if config == nil || config.BaseURL == "" { return nil, ErrInvalidConfiguration } @@ -139,6 +140,7 @@ func NewAdapter(config *Config, ingressAPIVersion string, ingressClassFilters [] ingressDefaultSecurityGroup: ingressDefaultSecurityGroup, ingressDefaultSSLPolicy: ingressDefaultSSLPolicy, ingressDefaultLoadBalancerType: loadBalancerTypesAWSToIngress[ingressDefaultLoadBalancerType], + ingressIpAddressType: ingressIpAddressType, clusterLocalDomain: clusterLocalDomain, routeGroupSupport: true, }, nil @@ -200,7 +202,7 @@ func (a *Adapter) newIngress(typ IngressType, metadata kubeItemMetadata, host st shared = false } - ipAddressType := aws.IPAddressTypeIPV4 + ipAddressType := a.ingressIpAddressType if getAnnotationsString(annotations, ingressALBIPAddressType, "") == aws.IPAddressTypeDualstack { ipAddressType = aws.IPAddressTypeDualstack } @@ -243,11 +245,6 @@ func (a *Adapter) newIngress(typ IngressType, metadata kubeItemMetadata, host st // convert to the internal naming e.g. nlb -> network loadBalancerType = loadBalancerTypesIngressToAWS[loadBalancerType] - if loadBalancerType == aws.LoadBalancerTypeNetwork { - // ensure ipv4 for network load balancers - ipAddressType = aws.IPAddressTypeIPV4 - } - http2 := true if getAnnotationsString(annotations, ingressHTTP2Annotation, "") == "false" { http2 = false diff --git a/kubernetes/adapter_test.go b/kubernetes/adapter_test.go index 730338b8..c0f1e9f9 100644 --- a/kubernetes/adapter_test.go +++ b/kubernetes/adapter_test.go @@ -443,7 +443,7 @@ func TestNewIngressFromKube(tt *testing.T) { }, } { tt.Run(tc.msg, func(t *testing.T) { - a, err := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testIngressDefaultSecurityGroup, testSSLPolicy, tc.defaultLoadBalancerType, DefaultClusterLocalDomain, false) + a, err := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testIngressDefaultSecurityGroup, testSSLPolicy, tc.defaultLoadBalancerType, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) if err != nil { t.Fatalf("cannot create kubernetes adapter: %v", err) } @@ -518,7 +518,7 @@ func (c *mockClient) patch(res string, payload []byte) (io.ReadCloser, error) { } func TestListIngress(t *testing.T) { - a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testIngressDefaultSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, false) + a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testIngressDefaultSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) client := &mockClient{} a.kubeClient = client ingresses, err := a.ListIngress() @@ -536,7 +536,7 @@ func TestListIngress(t *testing.T) { } func TestAdapterUpdateIngressLoadBalancer(t *testing.T) { - a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, false) + a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) client := &mockClient{} a.kubeClient = client ing := &Ingress{ @@ -565,7 +565,7 @@ func TestAdapterUpdateIngressLoadBalancer(t *testing.T) { } func TestUpdateRouteGroupLoadBalancer(t *testing.T) { - a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, false) + a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) client := &mockClient{} a.kubeClient = client ing := &Ingress{ @@ -604,7 +604,7 @@ func TestBrokenConfig(t *testing.T) { {"broken-cert", &Config{BaseURL: "dontcare", TLSClientConfig: TLSClientConfig{CAFile: "testdata/broken.pem"}}}, } { t.Run(fmt.Sprintf("%v", test.cfg), func(t *testing.T) { - _, err := NewAdapter(test.cfg, IngressAPIVersionNetworking, testIngressFilter, testSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, false) + _, err := NewAdapter(test.cfg, IngressAPIVersionNetworking, testIngressFilter, testSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) if err == nil { t.Error("expected an error") } @@ -613,7 +613,7 @@ func TestBrokenConfig(t *testing.T) { } func TestAdapter_GetConfigMap(t *testing.T) { - a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testIngressDefaultSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, false) + a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, testIngressFilter, testIngressDefaultSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) client := &mockClient{} a.kubeClient = client @@ -727,7 +727,7 @@ func TestListIngressFilterClass(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, test.ingressClassFilters, testIngressDefaultSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, false) + a, _ := NewAdapter(testConfig, IngressAPIVersionNetworking, test.ingressClassFilters, testIngressDefaultSecurityGroup, testSSLPolicy, aws.LoadBalancerTypeApplication, DefaultClusterLocalDomain, aws.DefaultIpAddressType, false) client := &mockClient{} a.kubeClient = client ingresses, err := a.ListResources() diff --git a/worker_test.go b/worker_test.go index 077eb7da..6c434ad8 100644 --- a/worker_test.go +++ b/worker_test.go @@ -469,6 +469,7 @@ func TestResourceConversionOneToOne(tt *testing.T) { sslPolicy, scenario.typeLB, clusterLocalDomain, + aws.DefaultIpAddressType, true) if err != nil { t.Fatal(err)