From d09d32de1f4eaabc304205a005f4080d717e8c44 Mon Sep 17 00:00:00 2001 From: "yogiii@amazon.com" Date: Thu, 31 Oct 2024 19:17:31 -0700 Subject: [PATCH 1/7] fixed documentation styling for Support UDP-based services over IPv6 --- docs/guide/service/annotations.md | 132 +++++++++++++++--------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/docs/guide/service/annotations.md b/docs/guide/service/annotations.md index c7ab9bc07..e9c494288 100644 --- a/docs/guide/service/annotations.md +++ b/docs/guide/service/annotations.md @@ -13,48 +13,48 @@ These annotations are specific to the kubernetes [service resources reconciled](#lb-type) by the AWS Load Balancer Controller. Although the list was initially derived from the k8s in-tree `kube-controller-manager`, this documentation is not an accurate reference for the services reconciled by the in-tree controller. -| Name | Type | Default | Notes | -|--------------------------------------------------------------------------------------------------|-------------------------|---------------------------|--------------------------------------------------------| -| [service.beta.kubernetes.io/load-balancer-source-ranges](#lb-source-ranges) | stringList | | | -| [service.beta.kubernetes.io/aws-load-balancer-security-group-prefix-lists](#lb-security-group-prefix-lists) | stringList | | | -| [service.beta.kubernetes.io/aws-load-balancer-type](#lb-type) | string | | | -| [service.beta.kubernetes.io/aws-load-balancer-nlb-target-type](#nlb-target-type) | string | | default `instance` in case of LoadBalancerClass | -| [service.beta.kubernetes.io/aws-load-balancer-name](#load-balancer-name) | string | | | -| [service.beta.kubernetes.io/aws-load-balancer-internal](#lb-internal) | boolean | false | deprecated, in favor of [aws-load-balancer-scheme](#lb-scheme)| -| [service.beta.kubernetes.io/aws-load-balancer-scheme](#lb-scheme) | string | internal | | -| [service.beta.kubernetes.io/aws-load-balancer-proxy-protocol](#proxy-protocol-v2) | string | | Set to `"*"` to enable | -| [service.beta.kubernetes.io/aws-load-balancer-ip-address-type](#ip-address-type) | string | ipv4 | ipv4 \| dualstack | -| [service.beta.kubernetes.io/aws-load-balancer-access-log-enabled](#deprecated-attributes) | boolean | false | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes)| -| [service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name](#deprecated-attributes) | string | | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes)| -| [service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix](#deprecated-attributes)| string | | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes)| -| [service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled](#deprecated-attributes)| boolean | false | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes)| -| [service.beta.kubernetes.io/aws-load-balancer-ssl-cert](#ssl-cert) | stringList | | | -| [service.beta.kubernetes.io/aws-load-balancer-ssl-ports](#ssl-ports) | stringList | | | -| [service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy](#ssl-negotiation-policy) | string | ELBSecurityPolicy-2016-08 | | -| [service.beta.kubernetes.io/aws-load-balancer-backend-protocol](#backend-protocol) | string | | | -| [service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags](#additional-resource-tags) | stringMap | | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol](#healthcheck-protocol) | string | TCP | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-port ](#healthcheck-port) | integer \| traffic-port | traffic-port | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-path](#healthcheck-path) | string | "/" for HTTP(S) protocols | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold](#healthcheck-healthy-threshold) | integer | 3 | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold](#healthcheck-unhealthy-threshold) | integer | 3 | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout](#healthcheck-timeout) | integer | 10 | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval](#healthcheck-interval) | integer | 10 | | -| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes](#healthcheck-success-codes) | string | 200-399 | | -| [service.beta.kubernetes.io/aws-load-balancer-eip-allocations](#eip-allocations) | stringList | | internet-facing lb only. Length must match the number of subnets| -| [service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses](#private-ipv4-addresses) | stringList | | internal lb only. Length must match the number of subnets | -| [service.beta.kubernetes.io/aws-load-balancer-ipv6-addresses](#ipv6-addresses) | stringList | | dualstack lb only. Length must match the number of subnets | -| [service.beta.kubernetes.io/aws-load-balancer-target-group-attributes](#target-group-attributes) | stringMap | | | -| [service.beta.kubernetes.io/aws-load-balancer-subnets](#subnets) | stringList | | | -| [service.beta.kubernetes.io/aws-load-balancer-alpn-policy](#alpn-policy) | string | | | -| [service.beta.kubernetes.io/aws-load-balancer-target-node-labels](#target-node-labels) | stringMap | | | -| [service.beta.kubernetes.io/aws-load-balancer-attributes](#load-balancer-attributes) | stringMap | | | -| [service.beta.kubernetes.io/aws-load-balancer-security-groups](#security-groups) | stringList | | | -| [service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules](#manage-backend-sg-rules) | boolean | true | If `service.beta.kubernetes.io/aws-load-balancer-security-groups` is specified, this must also be explicitly specified otherwise it defaults to `false`. | +| Name | Type | Default | Notes | +|--------------------------------------------------------------------------------------------------|-------------------------|---------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [service.beta.kubernetes.io/load-balancer-source-ranges](#lb-source-ranges) | stringList | | | +| [service.beta.kubernetes.io/aws-load-balancer-security-group-prefix-lists](#lb-security-group-prefix-lists) | stringList | | | +| [service.beta.kubernetes.io/aws-load-balancer-type](#lb-type) | string | | | +| [service.beta.kubernetes.io/aws-load-balancer-nlb-target-type](#nlb-target-type) | string | | default `instance` in case of LoadBalancerClass | +| [service.beta.kubernetes.io/aws-load-balancer-name](#load-balancer-name) | string | | | +| [service.beta.kubernetes.io/aws-load-balancer-internal](#lb-internal) | boolean | false | deprecated, in favor of [aws-load-balancer-scheme](#lb-scheme) | +| [service.beta.kubernetes.io/aws-load-balancer-scheme](#lb-scheme) | string | internal | | +| [service.beta.kubernetes.io/aws-load-balancer-proxy-protocol](#proxy-protocol-v2) | string | | Set to `"*"` to enable | +| [service.beta.kubernetes.io/aws-load-balancer-ip-address-type](#ip-address-type) | string | ipv4 | ipv4 \| dualstack | +| [service.beta.kubernetes.io/aws-load-balancer-access-log-enabled](#deprecated-attributes) | boolean | false | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes) | +| [service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name](#deprecated-attributes) | string | | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes) | +| [service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix](#deprecated-attributes)| string | | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes) | +| [service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled](#deprecated-attributes)| boolean | false | deprecated, in favor of [aws-load-balancer-attributes](#load-balancer-attributes) | +| [service.beta.kubernetes.io/aws-load-balancer-ssl-cert](#ssl-cert) | stringList | | | +| [service.beta.kubernetes.io/aws-load-balancer-ssl-ports](#ssl-ports) | stringList | | | +| [service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy](#ssl-negotiation-policy) | string | ELBSecurityPolicy-2016-08 | | +| [service.beta.kubernetes.io/aws-load-balancer-backend-protocol](#backend-protocol) | string | | | +| [service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags](#additional-resource-tags) | stringMap | | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol](#healthcheck-protocol) | string | TCP | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-port ](#healthcheck-port) | integer \| traffic-port | traffic-port | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-path](#healthcheck-path) | string | "/" for HTTP(S) protocols | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold](#healthcheck-healthy-threshold) | integer | 3 | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold](#healthcheck-unhealthy-threshold) | integer | 3 | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout](#healthcheck-timeout) | integer | 10 | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval](#healthcheck-interval) | integer | 10 | | +| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes](#healthcheck-success-codes) | string | 200-399 | | +| [service.beta.kubernetes.io/aws-load-balancer-eip-allocations](#eip-allocations) | stringList | | internet-facing lb only. Length must match the number of subnets | +| [service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses](#private-ipv4-addresses) | stringList | | internal lb only. Length must match the number of subnets | +| [service.beta.kubernetes.io/aws-load-balancer-ipv6-addresses](#ipv6-addresses) | stringList | | dualstack lb only. Length must match the number of subnets | +| [service.beta.kubernetes.io/aws-load-balancer-target-group-attributes](#target-group-attributes) | stringMap | | | +| [service.beta.kubernetes.io/aws-load-balancer-subnets](#subnets) | stringList | | | +| [service.beta.kubernetes.io/aws-load-balancer-alpn-policy](#alpn-policy) | string | | | +| [service.beta.kubernetes.io/aws-load-balancer-target-node-labels](#target-node-labels) | stringMap | | | +| [service.beta.kubernetes.io/aws-load-balancer-attributes](#load-balancer-attributes) | stringMap | | | +| [service.beta.kubernetes.io/aws-load-balancer-security-groups](#security-groups) | stringList | | | +| [service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules](#manage-backend-sg-rules) | boolean | true | If `service.beta.kubernetes.io/aws-load-balancer-security-groups` is specified, this must also be explicitly specified otherwise it defaults to `false`. | | [service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic](#update-security-settings) | string | | | [service.beta.kubernetes.io/aws-load-balancer-listener-attributes.${Protocol}-${Port}](#listener-attributes) | stringMap | | -| [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group) | boolean | false | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources. | -| [service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat](#enable-prefix-for-ipv6-source-nat) | string | off | Optional annotation. dualstack lb only. Allowed values - on and off | +| [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group) | boolean | false | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources. | +| [service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat](#enable-prefix-for-ipv6-source-nat) | string | off | Optional annotation. dualstack lb only. Allowed values - on and off | | [service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes](#source-nat-ipv6-prefixes) | stringList | | Optional annotation. dualstack lb only. This annotation is only applicable when user has to set the service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat to "on". Length must match the number of subnets | ## Traffic Routing @@ -206,35 +206,35 @@ You can configure dualstack NLB to support UDP-based services over IPv6 via the - service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat specifies whether Prefix for IPv6 source NAT is enabled or not. UDP-based support can be enabled for dualstack NLBs only if Prefix for IPv6 source NAT is enabled. - !!!note "" - - Applicable to Network Load Balancers using dualstack IP address type. - - This configuration is optional, and you can use it to enable UDP support over IPv6. - - Allowed values are either “on” or “off” - - Once the source prefix for source NATing is enabled, it cannot be disabled if load balancer has a UDP listener attached. - - Steps to disable the aws-load-balancer-enable-prefix-for-ipv6-source-nat after it is enabled and UDP listeners already attached. - - You will have to first remove the UDP listeners and apply the manifest. - - Update the manifest to set source NATing to "off" and then apply the manifest again. - - !!!example - - Enable prefix for IPv6 Source NAT - ``` - service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat: "on" - ``` + !!!note + - Applicable to Network Load Balancers using dualstack IP address type. + - This configuration is optional, and you can use it to enable UDP support over IPv6. + - Allowed values are either “on” or “off” + - Once the source prefix for source NATing is enabled, it cannot be disabled if load balancer has a UDP listener attached. + - Steps to disable the aws-load-balancer-enable-prefix-for-ipv6-source-nat after it is enabled and UDP listeners already attached. + - You will have to first remove the UDP listeners and apply the manifest. + - Update the manifest to set source NATing to "off" and then apply the manifest again. + + !!!example + - Enable prefix for IPv6 Source NAT + ``` + service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat: "on" + ``` - service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes specifies a list of IPv6 prefixes that should be used for IPv6 source NATing. - !!!note "" - - Applicable to Network Load Balancers using dualstack IP address type. - - This annotation can be specified only if service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat annotation is set to “on”. - - This configuration is optional and it can be used to specify custom IPv6 prefixes for IPv6 source NATing to support UDP based services routing in Network Load Balancers using dualstack IP address type. - - If service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat annotation is set to “on”, and you don’t specify this annotation, then IPv6 prefix/CIDR for source NATing will be auto-assigned to each subnet. - - If you are specifying this annotation, you must specify the same number of items in the list as the load balancer subnets annotation and following the same order. Each item in the list can have value of either “auto_assigned” or a valid IPv6 prefix/CIDR with prefix length of 80 and it should be in range of the corresponding subnet CIDR. - - Once the source NAT IPv6 prefixes are set, the IPv6 prefixes cannot be updated if the load balancer has a UDP listener attached. - - !!!example - ``` - service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes: 1025:0223:0009:6487:0001::/80, auto_assigned, 1025:0223:0010:6487:0001::/80 - ``` + !!!note + - Applicable to Network Load Balancers using dualstack IP address type. + - This annotation can be specified only if service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat annotation is set to “on”. + - This configuration is optional and it can be used to specify custom IPv6 prefixes for IPv6 source NATing to support UDP based services routing in Network Load Balancers using dualstack IP address type. + - If service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat annotation is set to “on”, and you don’t specify this annotation, then IPv6 prefix/CIDR for source NATing will be auto-assigned to each subnet. + - If you are specifying this annotation, you must specify the same number of items in the list as the load balancer subnets annotation and following the same order. Each item in the list can have value of either “auto_assigned” or a valid IPv6 prefix/CIDR with prefix length of 80 and it should be in range of the corresponding subnet CIDR. + - Once the source NAT IPv6 prefixes are set, the IPv6 prefixes cannot be updated if the load balancer has a UDP listener attached. + + !!!example + ``` + service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes: 1025:0223:0009:6487:0001::/80, auto_assigned, 1025:0223:0010:6487:0001::/80 + ``` ## Resource attributes NLB resource attributes can be controlled via the following annotations: From a2e02479398b9f637c978db5081bb1775d65798b Mon Sep 17 00:00:00 2001 From: wweiwei-li <79778352+wweiwei-li@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:35:21 -0800 Subject: [PATCH 2/7] Fix: order must match specified subnets (#3925) --- pkg/networking/subnet_resolver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/networking/subnet_resolver.go b/pkg/networking/subnet_resolver.go index aba7fba31..f00e8a51d 100644 --- a/pkg/networking/subnet_resolver.go +++ b/pkg/networking/subnet_resolver.go @@ -3,10 +3,11 @@ package networking import ( "context" "fmt" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "sort" "strings" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + awssdk "github.com/aws/aws-sdk-go-v2/aws" ec2sdk "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/go-logr/logr" @@ -328,7 +329,6 @@ func (r *defaultSubnetsResolver) ResolveViaNameOrIDSlice(ctx context.Context, su if err := r.validateSubnetsMinimalCount(resolvedSubnets, subnetLocale, resolveOpts); err != nil { return nil, err } - sortSubnetsByID(resolvedSubnets) return resolvedSubnets, nil } From 6aab160943b6a091e2063b4cc4fb2df843385cff Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 19 Nov 2024 13:40:08 -0800 Subject: [PATCH 3/7] trim control characters from secret to prevent newlines in client secret (#3936) --- pkg/ingress/model_build_actions.go | 2 +- pkg/ingress/model_build_actions_test.go | 55 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/pkg/ingress/model_build_actions.go b/pkg/ingress/model_build_actions.go index e2e883dbc..156229074 100644 --- a/pkg/ingress/model_build_actions.go +++ b/pkg/ingress/model_build_actions.go @@ -187,7 +187,7 @@ func (t *defaultModelBuildTask) buildAuthenticateOIDCAction(ctx context.Context, t.secretKeys = append(t.secretKeys, secretKey) clientID := strings.TrimRightFunc(string(rawClientID), unicode.IsSpace) - clientSecret := string(rawClientSecret) + clientSecret := strings.TrimRightFunc(string(rawClientSecret), unicode.IsControl) return elbv2model.Action{ Type: elbv2model.ActionTypeAuthenticateOIDC, AuthenticateOIDCConfig: &elbv2model.AuthenticateOIDCActionConfig{ diff --git a/pkg/ingress/model_build_actions_test.go b/pkg/ingress/model_build_actions_test.go index 91a3e5173..db24c7fe5 100644 --- a/pkg/ingress/model_build_actions_test.go +++ b/pkg/ingress/model_build_actions_test.go @@ -85,6 +85,61 @@ func Test_defaultModelBuildTask_buildAuthenticateOIDCAction(t *testing.T) { }, }, }, + { + name: "clientSecret has control characters at end", + env: env{ + secrets: []*corev1.Secret{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-ns", + Name: "my-k8s-secret", + }, + Data: map[string][]byte{ + "clientID": []byte("my-client-id"), + "clientSecret": []byte("my-client-secret\n"), + }, + }, + }, + }, + args: args{ + authCfg: AuthConfig{ + Type: AuthTypeCognito, + IDPConfigOIDC: &AuthIDPConfigOIDC{ + Issuer: "https://example.com", + AuthorizationEndpoint: "https://authorization.example.com", + TokenEndpoint: "https://token.example.com", + UserInfoEndpoint: "https://userinfo.example.co", + SecretName: "my-k8s-secret", + AuthenticationRequestExtraParams: map[string]string{ + "key1": "value1", + }, + }, + OnUnauthenticatedRequest: "authenticate", + Scope: "email", + SessionCookieName: "my-session-cookie", + SessionTimeout: 65536, + }, + namespace: "my-ns", + }, + want: elbv2model.Action{ + Type: elbv2model.ActionTypeAuthenticateOIDC, + AuthenticateOIDCConfig: &elbv2model.AuthenticateOIDCActionConfig{ + Issuer: "https://example.com", + AuthorizationEndpoint: "https://authorization.example.com", + TokenEndpoint: "https://token.example.com", + UserInfoEndpoint: "https://userinfo.example.co", + ClientID: "my-client-id", + ClientSecret: "my-client-secret", + AuthenticationRequestExtraParams: map[string]string{ + "key1": "value1", + }, + OnUnauthenticatedRequest: authBehaviorAuthenticate, + Scope: awssdk.String("email"), + SessionCookieName: awssdk.String("my-session-cookie"), + SessionTimeout: awssdk.Int64(65536), + }, + }, + }, { name: "clientID & clientSecret configured - legacy clientId", env: env{ From c701a42ce2e03c0dc18eea20449ef48d091192c8 Mon Sep 17 00:00:00 2001 From: Renjie Cai Date: Wed, 20 Nov 2024 08:22:33 +0800 Subject: [PATCH 4/7] fix: use target namespace to get pod info from repo (#3914) --- pkg/backend/endpoint_resolver.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/backend/endpoint_resolver.go b/pkg/backend/endpoint_resolver.go index 507ac16ca..55d7a50f6 100644 --- a/pkg/backend/endpoint_resolver.go +++ b/pkg/backend/endpoint_resolver.go @@ -160,7 +160,11 @@ func (r *defaultEndpointResolver) resolvePodEndpointsWithEndpointsData(ctx conte } epAddr := ep.Addresses[0] - podKey := types.NamespacedName{Namespace: svcKey.Namespace, Name: ep.TargetRef.Name} + podNamespace := svcKey.Namespace + if ep.TargetRef.Namespace != "" { + podNamespace = ep.TargetRef.Namespace + } + podKey := types.NamespacedName{Namespace: podNamespace, Name: ep.TargetRef.Name} pod, exists, err := r.podInfoRepo.Get(ctx, podKey) if err != nil { return nil, false, err From 13538cbe9bec2c019013631695237faacc116393 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 21 Nov 2024 08:02:48 -0800 Subject: [PATCH 5/7] FEAT: LBC metric collector (#3941) --- main.go | 13 +++- pkg/aws/cloud.go | 13 ++-- pkg/{aws/metrics => metrics/aws}/collector.go | 21 +++---- .../metrics => metrics/aws}/collector_test.go | 2 +- .../metrics => metrics/aws}/instruments.go | 34 +++------- pkg/metrics/lbc/collector.go | 39 ++++++++++++ pkg/metrics/lbc/instruments.go | 38 ++++++++++++ pkg/metrics/lbc/mockcollector.go | 37 +++++++++++ pkg/targetgroupbinding/resource_manager.go | 23 ++++--- .../resource_manager_test.go | 62 ++++++++++++------- 10 files changed, 205 insertions(+), 77 deletions(-) rename pkg/{aws/metrics => metrics/aws}/collector.go (92%) rename pkg/{aws/metrics => metrics/aws}/collector_test.go (98%) rename pkg/{aws/metrics => metrics/aws}/instruments.go (77%) create mode 100644 pkg/metrics/lbc/collector.go create mode 100644 pkg/metrics/lbc/instruments.go create mode 100644 pkg/metrics/lbc/mockcollector.go diff --git a/main.go b/main.go index b6484f24f..de2a3edc9 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,8 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/inject" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + awsmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" "sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding" @@ -81,7 +83,14 @@ func main() { ctrl.SetLogger(appLogger) klog.SetLoggerWithOptions(appLogger, klog.ContextualLogger(true)) - cloud, err := aws.NewCloud(controllerCFG.AWSConfig, metrics.Registry, ctrl.Log, nil) + var awsMetricsCollector *awsmetrics.Collector + lbcMetricsCollector := lbcmetrics.NewCollector(metrics.Registry) + + if metrics.Registry != nil { + awsMetricsCollector = awsmetrics.NewCollector(metrics.Registry) + } + + cloud, err := aws.NewCloud(controllerCFG.AWSConfig, awsMetricsCollector, ctrl.Log, nil) if err != nil { setupLog.Error(err, "unable to initialize AWS cloud") os.Exit(1) @@ -113,7 +122,7 @@ func main() { subnetResolver := networking.NewDefaultSubnetsResolver(azInfoProvider, cloud.EC2(), cloud.VpcID(), controllerCFG.ClusterName, ctrl.Log.WithName("subnets-resolver")) multiClusterManager := targetgroupbinding.NewMultiClusterManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log) tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.EC2(), - podInfoRepo, sgManager, sgReconciler, vpcInfoProvider, multiClusterManager, + podInfoRepo, sgManager, sgReconciler, vpcInfoProvider, multiClusterManager, lbcMetricsCollector, cloud.VpcID(), controllerCFG.ClusterName, controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules, controllerCFG.ServiceTargetENISGTags, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log) backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup, diff --git a/pkg/aws/cloud.go b/pkg/aws/cloud.go index cc679b943..41070e70d 100644 --- a/pkg/aws/cloud.go +++ b/pkg/aws/cloud.go @@ -11,7 +11,6 @@ import ( smithymiddleware "github.com/aws/smithy-go/middleware" "net" "os" - "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/metrics" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle" "sigs.k8s.io/aws-load-balancer-controller/pkg/version" "strings" @@ -21,11 +20,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/go-logr/logr" "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" amerrors "k8s.io/apimachinery/pkg/util/errors" epresolver "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/endpoints" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/provider" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + aws_metrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws" ) const userAgent = "elbv2.k8s.aws" @@ -60,7 +59,7 @@ type Cloud interface { } // NewCloud constructs new Cloud implementation. -func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (Cloud, error) { +func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (Cloud, error) { hasIPv4 := true addrs, err := net.InterfaceAddrs() if err == nil { @@ -122,12 +121,8 @@ func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger l }) } - if metricsRegisterer != nil { - metricsCollector, err := metrics.NewCollector(metricsRegisterer) - if err != nil { - return nil, errors.Wrapf(err, "failed to initialize sdk metrics collector") - } - awsConfig.APIOptions = metrics.WithSDKMetricCollector(metricsCollector, awsConfig.APIOptions) + if metricsCollector != nil { + awsConfig.APIOptions = aws_metrics.WithSDKMetricCollector(metricsCollector, awsConfig.APIOptions) } if awsClientsProvider == nil { diff --git a/pkg/aws/metrics/collector.go b/pkg/metrics/aws/collector.go similarity index 92% rename from pkg/aws/metrics/collector.go rename to pkg/metrics/aws/collector.go index 607d5f305..f6fe66e98 100644 --- a/pkg/aws/metrics/collector.go +++ b/pkg/metrics/aws/collector.go @@ -1,4 +1,4 @@ -package metrics +package aws import ( "context" @@ -18,24 +18,21 @@ const ( sdkMiddlewareCollectAPIRequestMetric = "collectAPIRequestMetric" ) -type collector struct { +type Collector struct { instruments *instruments } -func NewCollector(registerer prometheus.Registerer) (*collector, error) { - instruments, err := newInstruments(registerer) - if err != nil { - return nil, err - } - return &collector{ +func NewCollector(registerer prometheus.Registerer) *Collector { + instruments := newInstruments(registerer) + return &Collector{ instruments: instruments, - }, nil + } } /* WithSDKMetricCollector is a function that collects prometheus metrics for the AWS SDK Go v2 API calls ad requests */ -func WithSDKMetricCollector(c *collector, apiOptions []func(*smithymiddleware.Stack) error) []func(*smithymiddleware.Stack) error { +func WithSDKMetricCollector(c *Collector, apiOptions []func(*smithymiddleware.Stack) error) []func(*smithymiddleware.Stack) error { apiOptions = append(apiOptions, func(stack *smithymiddleware.Stack) error { return WithSDKCallMetricCollector(c)(stack) }, func(stack *smithymiddleware.Stack) error { @@ -48,7 +45,7 @@ func WithSDKMetricCollector(c *collector, apiOptions []func(*smithymiddleware.St WithSDKCallMetricCollector is a middleware for the AWS SDK Go v2 that collects and reports metrics on API calls. The call metrics are collected after the call is completed */ -func WithSDKCallMetricCollector(c *collector) func(stack *smithymiddleware.Stack) error { +func WithSDKCallMetricCollector(c *Collector) func(stack *smithymiddleware.Stack) error { return func(stack *smithymiddleware.Stack) error { return stack.Initialize.Add(smithymiddleware.InitializeMiddlewareFunc(sdkMiddlewareCollectAPICallMetric, func( ctx context.Context, input smithymiddleware.InitializeInput, next smithymiddleware.InitializeHandler, @@ -91,7 +88,7 @@ func WithSDKCallMetricCollector(c *collector) func(stack *smithymiddleware.Stack WithSDKRequestMetricCollector is a middleware for the AWS SDK Go v2 that collects and reports metrics on API requests. The request metrics are collected after each retry attempts */ -func WithSDKRequestMetricCollector(c *collector) func(stack *smithymiddleware.Stack) error { +func WithSDKRequestMetricCollector(c *Collector) func(stack *smithymiddleware.Stack) error { return func(stack *smithymiddleware.Stack) error { return stack.Finalize.Add(smithymiddleware.FinalizeMiddlewareFunc(sdkMiddlewareCollectAPIRequestMetric, func( ctx context.Context, input smithymiddleware.FinalizeInput, next smithymiddleware.FinalizeHandler, diff --git a/pkg/aws/metrics/collector_test.go b/pkg/metrics/aws/collector_test.go similarity index 98% rename from pkg/aws/metrics/collector_test.go rename to pkg/metrics/aws/collector_test.go index db2579381..3e52a2dda 100644 --- a/pkg/aws/metrics/collector_test.go +++ b/pkg/metrics/aws/collector_test.go @@ -1,4 +1,4 @@ -package metrics +package aws import ( "errors" diff --git a/pkg/aws/metrics/instruments.go b/pkg/metrics/aws/instruments.go similarity index 77% rename from pkg/aws/metrics/instruments.go rename to pkg/metrics/aws/instruments.go index e3ca81204..150bf1101 100644 --- a/pkg/aws/metrics/instruments.go +++ b/pkg/metrics/aws/instruments.go @@ -1,11 +1,11 @@ -package metrics +package aws import ( "github.com/prometheus/client_golang/prometheus" ) const ( - metricSubsystemAWS = "aws" + metricSubSystem = "aws" metricAPICallsTotal = "api_calls_total" metricAPICallDurationSeconds = "api_call_duration_seconds" @@ -31,55 +31,41 @@ type instruments struct { } // newInstruments allocates and register new metrics to registerer -func newInstruments(registerer prometheus.Registerer) (*instruments, error) { +func newInstruments(registerer prometheus.Registerer) *instruments { apiCallsTotal := prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPICallsTotal, Help: "Total number of SDK API calls from the customer's code to AWS services", }, []string{labelService, labelOperation, labelStatusCode, labelErrorCode}) apiCallDurationSeconds := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPICallDurationSeconds, Help: "Perceived latency from when your code makes an SDK call, includes retries", }, []string{labelService, labelOperation}) apiCallRetries := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPICallRetries, Help: "Number of times the SDK retried requests to AWS services for SDK API calls", Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, }, []string{labelService, labelOperation}) apiRequestsTotal := prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPIRequestsTotal, Help: "Total number of HTTP requests that the SDK made", }, []string{labelService, labelOperation, labelStatusCode, labelErrorCode}) apiRequestDurationSecond := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPIRequestDurationSeconds, Help: "Latency of an individual HTTP request to the service endpoint", }, []string{labelService, labelOperation}) - if err := registerer.Register(apiCallsTotal); err != nil { - return nil, err - } - if err := registerer.Register(apiCallDurationSeconds); err != nil { - return nil, err - } - if err := registerer.Register(apiCallRetries); err != nil { - return nil, err - } - if err := registerer.Register(apiRequestsTotal); err != nil { - return nil, err - } - if err := registerer.Register(apiRequestDurationSecond); err != nil { - return nil, err - } + registerer.MustRegister(apiCallsTotal, apiCallDurationSeconds, apiCallRetries, apiRequestsTotal, apiRequestDurationSecond) return &instruments{ apiCallsTotal: apiCallsTotal, apiCallDurationSeconds: apiCallDurationSeconds, apiCallRetries: apiCallRetries, apiRequestsTotal: apiRequestsTotal, apiRequestDurationSecond: apiRequestDurationSecond, - }, nil + } } diff --git a/pkg/metrics/lbc/collector.go b/pkg/metrics/lbc/collector.go new file mode 100644 index 000000000..34da12848 --- /dev/null +++ b/pkg/metrics/lbc/collector.go @@ -0,0 +1,39 @@ +package lbc + +import ( + "github.com/prometheus/client_golang/prometheus" + "time" +) + +type MetricCollector interface { + // ObservePodReadinessGateReady this metric is useful to determine how fast pods are becoming ready in the load balancer. + // Due to some architectural constraints, we can only emit this metric for pods that are using readiness gates. + ObservePodReadinessGateReady(namespace string, tgbName string, duration time.Duration) +} + +type collector struct { + instruments *instruments +} + +type noOpCollector struct{} + +func (n *noOpCollector) ObservePodReadinessGateReady(_ string, _ string, _ time.Duration) { +} + +func NewCollector(registerer prometheus.Registerer) MetricCollector { + if registerer == nil { + return &noOpCollector{} + } + + instruments := newInstruments(registerer) + return &collector{ + instruments: instruments, + } +} + +func (c *collector) ObservePodReadinessGateReady(namespace string, tgbName string, duration time.Duration) { + c.instruments.podReadinessFlipSeconds.With(prometheus.Labels{ + labelNamespace: namespace, + labelName: tgbName, + }).Observe(duration.Seconds()) +} diff --git a/pkg/metrics/lbc/instruments.go b/pkg/metrics/lbc/instruments.go new file mode 100644 index 000000000..0a38b7f77 --- /dev/null +++ b/pkg/metrics/lbc/instruments.go @@ -0,0 +1,38 @@ +package lbc + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + metricSubsystem = "awslbc" +) + +// These metrics are exported to be used in unit test validation. +const ( + // MetricPodReadinessGateReady tracks the time to flip a readiness gate to true + MetricPodReadinessGateReady = "readiness_gate_ready_seconds" +) + +const ( + labelNamespace = "namespace" + labelName = "name" +) + +type instruments struct { + podReadinessFlipSeconds *prometheus.HistogramVec +} + +// newInstruments allocates and register new metrics to registerer +func newInstruments(registerer prometheus.Registerer) *instruments { + podReadinessFlipSeconds := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Subsystem: metricSubsystem, + Name: MetricPodReadinessGateReady, + Help: "Latency from pod getting added to the load balancer until the readiness gate is flipped to healthy.", + }, []string{labelNamespace, labelName}) + + registerer.MustRegister(podReadinessFlipSeconds) + return &instruments{ + podReadinessFlipSeconds: podReadinessFlipSeconds, + } +} diff --git a/pkg/metrics/lbc/mockcollector.go b/pkg/metrics/lbc/mockcollector.go new file mode 100644 index 000000000..9c8fb6a43 --- /dev/null +++ b/pkg/metrics/lbc/mockcollector.go @@ -0,0 +1,37 @@ +package lbc + +import ( + "time" +) + +type MockCollector struct { + Invocations map[string][]interface{} +} + +type MockHistogramMetric struct { + namespace string + name string + duration time.Duration +} + +func (m *MockCollector) ObservePodReadinessGateReady(namespace string, tgbName string, d time.Duration) { + m.recordHistogram(MetricPodReadinessGateReady, namespace, tgbName, d) +} + +func (m *MockCollector) recordHistogram(metricName string, namespace string, name string, d time.Duration) { + m.Invocations[metricName] = append(m.Invocations[MetricPodReadinessGateReady], MockHistogramMetric{ + namespace: namespace, + name: name, + duration: d, + }) +} + +func NewMockCollector() MetricCollector { + + mockInvocations := make(map[string][]interface{}) + mockInvocations[MetricPodReadinessGateReady] = make([]interface{}, 0) + + return &MockCollector{ + Invocations: mockInvocations, + } +} diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 9056c3666..af25a824f 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -6,6 +6,7 @@ import ( elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/aws/smithy-go" "net/netip" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" "time" "k8s.io/client-go/tools/record" @@ -37,7 +38,7 @@ type ResourceManager interface { // NewDefaultResourceManager constructs new defaultResourceManager. func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELBV2, ec2Client services.EC2, podInfoRepo k8s.PodInfoRepo, sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler, - vpcInfoProvider networking.VPCInfoProvider, multiClusterManager MultiClusterManager, + vpcInfoProvider networking.VPCInfoProvider, multiClusterManager MultiClusterManager, metricsCollector lbcmetrics.MetricCollector, vpcID string, clusterName string, failOpenEnabled bool, endpointSliceEnabled bool, disabledRestrictedSGRulesFlag bool, endpointSGTags map[string]string, eventRecorder record.EventRecorder, logger logr.Logger) *defaultResourceManager { @@ -60,6 +61,7 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB vpcInfoProvider: vpcInfoProvider, podInfoRepo: podInfoRepo, multiClusterManager: multiClusterManager, + metricsCollector: metricsCollector, requeueDuration: defaultRequeueDuration, } @@ -78,6 +80,7 @@ type defaultResourceManager struct { vpcInfoProvider networking.VPCInfoProvider podInfoRepo k8s.PodInfoRepo multiClusterManager MultiClusterManager + metricsCollector lbcmetrics.MetricCollector vpcID string requeueDuration time.Duration @@ -240,7 +243,7 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, return "", "", false, err } - anyPodNeedFurtherProbe, err := m.updateTargetHealthPodCondition(ctx, targetHealthCondType, matchedEndpointAndTargets, unmatchedEndpoints) + anyPodNeedFurtherProbe, err := m.updateTargetHealthPodCondition(ctx, targetHealthCondType, matchedEndpointAndTargets, unmatchedEndpoints, tgb) if err != nil { return "", "", false, err } @@ -374,13 +377,13 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a // updateTargetHealthPodCondition will updates pod's targetHealth condition for matchedEndpointAndTargets and unmatchedEndpoints. // returns whether further probe is needed or not func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Context, targetHealthCondType corev1.PodConditionType, - matchedEndpointAndTargets []podEndpointAndTargetPair, unmatchedEndpoints []backend.PodEndpoint) (bool, error) { + matchedEndpointAndTargets []podEndpointAndTargetPair, unmatchedEndpoints []backend.PodEndpoint, tgb *elbv2api.TargetGroupBinding) (bool, error) { anyPodNeedFurtherProbe := false for _, endpointAndTarget := range matchedEndpointAndTargets { pod := endpointAndTarget.endpoint.Pod targetHealth := endpointAndTarget.target.TargetHealth - needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType, tgb) if err != nil { return false, err } @@ -396,7 +399,7 @@ func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Cont Reason: elbv2types.TargetHealthReasonEnumRegistrationInProgress, Description: awssdk.String("Target registration is in progress"), } - needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType, tgb) if err != nil { return false, err } @@ -410,7 +413,7 @@ func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Cont // updateTargetHealthPodConditionForPod updates pod's targetHealth condition for a single pod and its matched target. // returns whether further probe is needed or not. func (m *defaultResourceManager) updateTargetHealthPodConditionForPod(ctx context.Context, pod k8s.PodInfo, - targetHealth *elbv2types.TargetHealth, targetHealthCondType corev1.PodConditionType) (bool, error) { + targetHealth *elbv2types.TargetHealth, targetHealthCondType corev1.PodConditionType, tgb *elbv2api.TargetGroupBinding) (bool, error) { if !pod.HasAnyOfReadinessGates([]corev1.PodConditionType{targetHealthCondType}) { return false, nil } @@ -468,6 +471,12 @@ func (m *defaultResourceManager) updateTargetHealthPodConditionForPod(ctx contex return false, err } + // Only update duration on unhealthy -> healthy flips. + if targetHealthCondStatus == corev1.ConditionTrue && hasExistingTargetHealthCond && !existingTargetHealthCond.LastTransitionTime.IsZero() && existingTargetHealthCond.Status != corev1.ConditionTrue { + delta := newTargetHealthCond.LastTransitionTime.Sub(existingTargetHealthCond.LastTransitionTime.Time) + m.metricsCollector.ObservePodReadinessGateReady(tgb.Namespace, tgb.Name, delta) + } + return needFurtherProbe, nil } @@ -509,7 +518,7 @@ func (m *defaultResourceManager) updatePodAsHealthyForDeletedTGB(ctx context.Con State: elbv2types.TargetHealthStateEnumHealthy, Description: awssdk.String("Target Group Binding is deleted"), } - _, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + _, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType, tgb) if err != nil { return err } diff --git a/pkg/targetgroupbinding/resource_manager_test.go b/pkg/targetgroupbinding/resource_manager_test.go index 1031c1e53..4839b2fb2 100644 --- a/pkg/targetgroupbinding/resource_manager_test.go +++ b/pkg/targetgroupbinding/resource_manager_test.go @@ -3,6 +3,8 @@ package targetgroupbinding import ( "context" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" "testing" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -26,6 +28,9 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing pods []*corev1.Pod } + tgbName := "tgb" + tgbNamespace := "tgbNamespace" + type args struct { pod k8s.PodInfo targetHealth *elbv2types.TargetHealth @@ -33,12 +38,13 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing } tests := []struct { - name string - env env - args args - want bool - wantPod *corev1.Pod - wantErr error + name string + env env + args args + want bool + wantPod *corev1.Pod + wantMetric bool + wantErr error }{ { name: "pod contains readinessGate and targetHealth is healthy - add pod condition", @@ -137,10 +143,11 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing Status: corev1.PodStatus{ Conditions: []corev1.PodCondition{ { - Type: "target-health.elbv2.k8s.aws/my-tgb", - Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), - Reason: "Elb.RegistrationInProgress", - Status: corev1.ConditionFalse, + Type: "target-health.elbv2.k8s.aws/my-tgb", + Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), + Reason: "Elb.RegistrationInProgress", + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), }, { Type: corev1.ContainersReady, @@ -162,10 +169,11 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing }, Conditions: []corev1.PodCondition{ { - Type: "target-health.elbv2.k8s.aws/my-tgb", - Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), - Reason: "Elb.RegistrationInProgress", - Status: corev1.ConditionFalse, + Type: "target-health.elbv2.k8s.aws/my-tgb", + Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), + Reason: "Elb.RegistrationInProgress", + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), }, { Type: corev1.ContainersReady, @@ -178,7 +186,8 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing }, targetHealthCondType: "target-health.elbv2.k8s.aws/my-tgb", }, - want: false, + want: false, + wantMetric: true, wantPod: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -226,10 +235,11 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing Status: corev1.PodStatus{ Conditions: []corev1.PodCondition{ { - Type: "target-health.elbv2.k8s.aws/my-tgb", - Status: corev1.ConditionFalse, - Reason: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), - Message: "Target registration is in progress", + Type: "target-health.elbv2.k8s.aws/my-tgb", + Status: corev1.ConditionFalse, + Reason: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), + Message: "Target registration is in progress", + LastTransitionTime: metav1.Now(), }, { Type: corev1.ContainersReady, @@ -382,10 +392,15 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing k8sClient := testclient.NewClientBuilder().WithScheme(k8sSchema).Build() m := &defaultResourceManager{ - k8sClient: k8sClient, - logger: logr.New(&log.NullLogSink{}), + k8sClient: k8sClient, + logger: logr.New(&log.NullLogSink{}), + metricsCollector: lbcmetrics.NewMockCollector(), } + tgb := &elbv2api.TargetGroupBinding{} + tgb.Name = tgbName + tgb.Namespace = tgbNamespace + ctx := context.Background() for _, pod := range tt.env.pods { err := k8sClient.Create(ctx, pod.DeepCopy()) @@ -393,7 +408,7 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing } got, err := m.updateTargetHealthPodConditionForPod(context.Background(), - tt.args.pod, tt.args.targetHealth, tt.args.targetHealthCondType) + tt.args.pod, tt.args.targetHealth, tt.args.targetHealthCondType, tgb) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -410,6 +425,9 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing } assert.True(t, cmp.Equal(tt.wantPod, updatedPod, opts), "diff", cmp.Diff(tt.wantPod, updatedPod, opts)) } + + mockCollector := m.metricsCollector.(*lbcmetrics.MockCollector) + assert.Equal(t, tt.wantMetric, len(mockCollector.Invocations[lbcmetrics.MetricPodReadinessGateReady]) == 1) }) } } From 8ba34e27224d1f7a56f0e416086c43b6877c7e65 Mon Sep 17 00:00:00 2001 From: wweiwei-li <79778352+wweiwei-li@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:05:29 -0800 Subject: [PATCH 6/7] enable http and https listener attributes (#3948) --- docs/guide/ingress/annotations.md | 9 ++ pkg/deploy/elbv2/listener_manager.go | 4 +- pkg/ingress/model_build_listener.go | 29 +++++-- pkg/ingress/model_build_listener_test.go | 84 +++++++++++++++++++ .../model_build_load_balancer_attributes.go | 28 +++++-- ...del_build_load_balancer_attributes_test.go | 5 +- test/e2e/ingress/vanilla_ingress_test.go | 62 ++++++++++++++ 7 files changed, 201 insertions(+), 20 deletions(-) diff --git a/docs/guide/ingress/annotations.md b/docs/guide/ingress/annotations.md index a47e53405..2dc2947ed 100644 --- a/docs/guide/ingress/annotations.md +++ b/docs/guide/ingress/annotations.md @@ -60,6 +60,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t | [alb.ingress.kubernetes.io/target-node-labels](#target-node-labels) | stringMap |N/A| Ingress,Service | N/A | | [alb.ingress.kubernetes.io/mutual-authentication](#mutual-authentication) | json |N/A| Ingress | Exclusive | | [alb.ingress.kubernetes.io/multi-cluster-target-group](#multi-cluster-target-group) | boolean |N/A| Ingress, Service | N/A | +| [alb.ingress.kubernetes.io/listener-attributes.${Protocol}-${Port}](#listener-attributes) | stringMap |N/A| Ingress |Merge| ## IngressGroup IngressGroup feature enables you to group multiple Ingress resources together. @@ -903,6 +904,14 @@ Custom attributes to LoadBalancers and TargetGroups can be controlled with follo alb.ingress.kubernetes.io/multi-cluster-target-group: "true" ``` +- `alb.ingress.kubernetes.io/listener-attributes.${Protocol}-${Port}` specifies Listener Attributes which should be applied to listener. + + !!!example + - Server header enablement attribute + ``` + alb.ingress.kubernetes.io/listener-attributes.HTTP-80: routing.http.response.server.enabled=true + ``` + ## Resource Tags The AWS Load Balancer Controller automatically applies following tags to the AWS resources (ALB/TargetGroups/SecurityGroups/Listener/ListenerRule) it creates: diff --git a/pkg/deploy/elbv2/listener_manager.go b/pkg/deploy/elbv2/listener_manager.go index 1aa4a6b70..a9db44077 100644 --- a/pkg/deploy/elbv2/listener_manager.go +++ b/pkg/deploy/elbv2/listener_manager.go @@ -23,8 +23,8 @@ import ( ) var PROTOCOLS_SUPPORTING_LISTENER_ATTRIBUTES = map[elbv2model.Protocol]bool{ - elbv2model.ProtocolHTTP: false, - elbv2model.ProtocolHTTPS: false, + elbv2model.ProtocolHTTP: true, + elbv2model.ProtocolHTTPS: true, elbv2model.ProtocolTCP: true, elbv2model.ProtocolUDP: false, elbv2model.ProtocolTLS: false, diff --git a/pkg/ingress/model_build_listener.go b/pkg/ingress/model_build_listener.go index 31757773e..80f848ba3 100644 --- a/pkg/ingress/model_build_listener.go +++ b/pkg/ingress/model_build_listener.go @@ -426,6 +426,14 @@ func (t *defaultModelBuildTask) fetchTrustStoreArnFromName(ctx context.Context, func (t *defaultModelBuildTask) buildIngressGroupListenerAttributes(ctx context.Context, ingList []ClassifiedIngress, listenerProtocol elbv2model.Protocol, port int32) ([]elbv2model.ListenerAttribute, error) { rawIngGrouplistenerAttributes := make(map[string]string) + ingClassAttributes := make(map[string]string) + if len(ingList) > 0 { + var err error + ingClassAttributes, err = t.buildIngressClassListenerAttributes(ingList[0].IngClassConfig, listenerProtocol, port) + if err != nil { + return nil, err + } + } for _, ing := range ingList { ingAttributes, err := t.buildIngressListenerAttributes(ctx, ing.Ing.Annotations, port, listenerProtocol) if err != nil { @@ -435,18 +443,23 @@ func (t *defaultModelBuildTask) buildIngressGroupListenerAttributes(ctx context. attributeKey := attribute.Key attributeValue := attribute.Value if existingAttributeValue, exists := rawIngGrouplistenerAttributes[attributeKey]; exists && existingAttributeValue != attributeValue { - return nil, errors.Errorf("conflicting attributes %v: %v | %v", attributeKey, existingAttributeValue, attributeValue) + if ingClassValue, exists := ingClassAttributes[attributeKey]; exists { + // Conflict is resolved by ingClassAttributes, show a warning + t.logger.Info("listener attribute conflict resolved by ingress class", + "attributeKey", attributeKey, + "existingValue", existingAttributeValue, + "newValue", attributeValue, + "ingClassValue", ingClassValue) + } else { + // Conflict is not resolved by ingClassAttributes, return an error + return nil, errors.Errorf("conflicting listener attributes %v: %v | %v for ingress %s/%s", + attributeKey, existingAttributeValue, attributeValue, ing.Ing.Namespace, ing.Ing.Name) + } } rawIngGrouplistenerAttributes[attributeKey] = attributeValue } } - if len(ingList) > 0 { - ingClassAttributes, err := t.buildIngressClassListenerAttributes(ingList[0].IngClassConfig, listenerProtocol, port) - if err != nil { - return nil, err - } - rawIngGrouplistenerAttributes = algorithm.MergeStringMap(ingClassAttributes, rawIngGrouplistenerAttributes) - } + rawIngGrouplistenerAttributes = algorithm.MergeStringMap(ingClassAttributes, rawIngGrouplistenerAttributes) attributes := make([]elbv2model.ListenerAttribute, 0, len(rawIngGrouplistenerAttributes)) for attrKey, attrValue := range rawIngGrouplistenerAttributes { attributes = append(attributes, elbv2model.ListenerAttribute{ diff --git a/pkg/ingress/model_build_listener_test.go b/pkg/ingress/model_build_listener_test.go index ca856b6d9..9514b8f6e 100644 --- a/pkg/ingress/model_build_listener_test.go +++ b/pkg/ingress/model_build_listener_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" @@ -272,6 +273,89 @@ func Test_buildListenerAttributes(t *testing.T) { }, }, }, + { + name: "Ignore conflicting value when the key is specified by ingress class param", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-6", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "attrKey1=attrValue1", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + Listeners: []elbv2api.Listener{ + { + Protocol: "HTTP", + Port: 80, + ListenerAttributes: []elbv2api.Attribute{ + { + Key: "attrKey1", + Value: "attrValue1", + }, + }, + }, + }, + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-7", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "attrKey1=attrValue2", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + Listeners: []elbv2api.Listener{ + { + Protocol: "HTTP", + Port: 80, + ListenerAttributes: []elbv2api.Attribute{ + { + Key: "attrKey1", + Value: "attrValue1", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantErr: false, + wantValue: []elbv2model.ListenerAttribute{ + { + Key: "attrKey1", + Value: "attrValue1", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/ingress/model_build_load_balancer_attributes.go b/pkg/ingress/model_build_load_balancer_attributes.go index e63e286bb..af67c4923 100644 --- a/pkg/ingress/model_build_load_balancer_attributes.go +++ b/pkg/ingress/model_build_load_balancer_attributes.go @@ -9,6 +9,14 @@ import ( // buildIngressGroupLoadBalancerAttributes builds the LB attributes for a group of Ingresses. func (t *defaultModelBuildTask) buildIngressGroupLoadBalancerAttributes(ingList []ClassifiedIngress) (map[string]string, error) { ingGroupAttributes := make(map[string]string) + ingClassAttributes := make(map[string]string) + if len(ingList) > 0 { + var err error + ingClassAttributes, err = t.buildIngressClassLoadBalancerAttributes(ingList[0].IngClassConfig) + if err != nil { + return nil, err + } + } for _, ing := range ingList { ingAttributes, err := t.buildIngressLoadBalancerAttributes(ing) if err != nil { @@ -18,18 +26,22 @@ func (t *defaultModelBuildTask) buildIngressGroupLoadBalancerAttributes(ingList for attrKey, attrValue := range ingAttributes { existingAttrValue, exists := ingGroupAttributes[attrKey] if exists && existingAttrValue != attrValue { - return nil, errors.Errorf("conflicting attributes %v: %v | %v", attrKey, existingAttrValue, attrValue) + if ingClassValue, exists := ingClassAttributes[attrKey]; exists { + // Conflict is resolved by ingClassAttributes, show a warning + t.logger.Info("load balancer attribute conflict resolved by ingress class", + "attributeKey", attrKey, + "existingValue", existingAttrValue, + "newValue", attrValue, + "ingClassValue", ingClassValue) + } else { + // Conflict is not resolved by ingClassAttributes, return an error + return nil, errors.Errorf("conflicting load balancer attributes %v: %v | %v", attrKey, existingAttrValue, attrValue) + } } ingGroupAttributes[attrKey] = attrValue } } - if len(ingList) > 0 { - ingClassAttributes, err := t.buildIngressClassLoadBalancerAttributes(ingList[0].IngClassConfig) - if err != nil { - return nil, err - } - return algorithm.MergeStringMap(ingClassAttributes, ingGroupAttributes), nil - } + ingGroupAttributes = algorithm.MergeStringMap(ingClassAttributes, ingGroupAttributes) return ingGroupAttributes, nil } diff --git a/pkg/ingress/model_build_load_balancer_attributes_test.go b/pkg/ingress/model_build_load_balancer_attributes_test.go index bb42e50e3..4c623bd8e 100644 --- a/pkg/ingress/model_build_load_balancer_attributes_test.go +++ b/pkg/ingress/model_build_load_balancer_attributes_test.go @@ -2,13 +2,14 @@ package ingress import ( "fmt" + "testing" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" - "testing" ) func Test_defaultModelBuildTask_buildIngressGroupLoadBalancerAttributes(t *testing.T) { @@ -82,7 +83,7 @@ func Test_defaultModelBuildTask_buildIngressGroupLoadBalancerAttributes(t *testi }, }, }, - wantErr: errors.New("conflicting attributes deletion_protection.enabled: true | false"), + wantErr: errors.New("conflicting load balancer attributes deletion_protection.enabled: true | false"), }, { name: "non-empty annotation attributes from single Ingress, non-empty IngressClass attributes - has overlap attributes", diff --git a/test/e2e/ingress/vanilla_ingress_test.go b/test/e2e/ingress/vanilla_ingress_test.go index f5366889d..a79a3e673 100644 --- a/test/e2e/ingress/vanilla_ingress_test.go +++ b/test/e2e/ingress/vanilla_ingress_test.go @@ -11,6 +11,7 @@ import ( "github.com/gavv/httpexpect/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -777,6 +778,56 @@ var _ = Describe("vanilla ingress tests", func() { } }) }) + + Context("with `alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", + } + if tf.Options.IPFamily == "IPv6" { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) + + defer resStack.TearDown(ctx) + + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + sdkListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, lbARN) + + Eventually(func() bool { + return verifyListenerAttributes(ctx, tf, *sdkListeners[0].ListenerArn, map[string]string{ + "routing.http.response.server.enabled": "false", + }) == nil + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) + }) }) // ExpectOneLBProvisionedForIngress expects one LoadBalancer provisioned for Ingress. @@ -820,3 +871,14 @@ func ExpectLBDNSBeAvailable(ctx context.Context, tf *framework.Framework, lbARN Expect(err).NotTo(HaveOccurred()) tf.Logger.Info("dns becomes available", "dns", lbDNS) } + +func verifyListenerAttributes(ctx context.Context, f *framework.Framework, lsARN string, expectedAttrs map[string]string) error { + lsAttrs, err := f.LBManager.GetListenerAttributes(ctx, lsARN) + Expect(err).NotTo(HaveOccurred()) + for _, attr := range lsAttrs { + if val, ok := expectedAttrs[awssdk.ToString(attr.Key)]; ok && val != awssdk.ToString(attr.Value) { + return errors.Errorf("Attribute %v, expected %v, actual %v", awssdk.ToString(attr.Key), val, awssdk.ToString(attr.Value)) + } + } + return nil +} From 35c3000dc9c8c99cb26f3cf4ccc41c2130e97f51 Mon Sep 17 00:00:00 2001 From: Weiwei Li Date: Thu, 21 Nov 2024 14:16:41 -0800 Subject: [PATCH 7/7] cut v2.10.1 release --- Makefile | 2 +- config/controller/kustomization.yaml | 2 +- docs/deploy/installation.md | 16 ++++++++-------- docs/examples/echo_server.md | 10 +++++----- helm/aws-load-balancer-controller/Chart.yaml | 4 ++-- helm/aws-load-balancer-controller/test.yaml | 2 +- helm/aws-load-balancer-controller/values.yaml | 2 +- version-stable.txt | 2 +- version.txt | 2 +- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 05c8ae4a4..9e808fc92 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ MAKEFILE_PATH = $(dir $(realpath -s $(firstword $(MAKEFILE_LIST)))) # Image URL to use all building/pushing image targets -IMG ?= public.ecr.aws/eks/aws-load-balancer-controller:v2.10.0 +IMG ?= public.ecr.aws/eks/aws-load-balancer-controller:v2.10.1 # Image URL to use for builder stage in Docker build GOLANG_VERSION ?= $(shell cat .go-version) BUILD_IMAGE ?= public.ecr.aws/docker/library/golang:$(GOLANG_VERSION) diff --git a/config/controller/kustomization.yaml b/config/controller/kustomization.yaml index 0b017c773..f8b3b8828 100644 --- a/config/controller/kustomization.yaml +++ b/config/controller/kustomization.yaml @@ -9,4 +9,4 @@ kind: Kustomization images: - name: controller newName: public.ecr.aws/eks/aws-load-balancer-controller - newTag: v2.10.0 + newTag: v2.10.1 diff --git a/docs/deploy/installation.md b/docs/deploy/installation.md index b0fbee642..03e6084b7 100644 --- a/docs/deploy/installation.md +++ b/docs/deploy/installation.md @@ -90,15 +90,15 @@ Example condition for cluster name resource tag: 2. Download an IAM policy for the LBC using one of the following commands:

If your cluster is in a US Gov Cloud region: ``` - curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/install/iam_policy_us-gov.json + curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/install/iam_policy_us-gov.json ``` If your cluster is in a China region: ``` - curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/install/iam_policy_cn.json + curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/install/iam_policy_cn.json ``` If your cluster is in any other region: ``` - curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/install/iam_policy.json + curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/install/iam_policy.json ``` 3. Create an IAM policy named `AWSLoadBalancerControllerIAMPolicy`. If you downloaded a different policy, replace `iam-policy` with the name of the policy that you downloaded. @@ -124,7 +124,7 @@ Example condition for cluster name resource tag: ### Option B: Attach IAM policies to nodes If you're not setting up IAM roles for service accounts, apply the IAM policies from the following URL at a minimum. Please be aware of the possibility that the controller permissions may be assumed by other users in a pod after retrieving the node role credentials, so the best practice would be using IRSA instead of attaching IAM policy directly. ``` -curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/install/iam_policy.json +curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/install/iam_policy.json ``` The following IAM permissions subset is for those using `TargetGroupBinding` only and don't plan to use the LBC to manage security group rules: @@ -209,7 +209,7 @@ We recommend using the Helm chart to install the controller. The chart supports ### Apply YAML 1. Download the spec for the LBC. ``` - wget https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.10.0/v2_10_0_full.yaml + wget https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.10.1/v2_10_1_full.yaml ``` 2. Edit the saved yaml file, go to the Deployment spec, and set the controller `--cluster-name` arg value to your EKS cluster name ``` @@ -233,15 +233,15 @@ We recommend using the Helm chart to install the controller. The chart supports ``` 4. Apply the yaml file ``` - kubectl apply -f v2_10_0_full.yaml + kubectl apply -f v2_10_1_full.yaml ``` 5. Optionally download the default ingressclass and ingressclass params ``` - wget https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.10.0/v2_10_0_ingclass.yaml + wget https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.10.1/v2_10_1_ingclass.yaml ``` 6. Apply the ingressclass and params ``` - kubectl apply -f v2_10_0_ingclass.yaml + kubectl apply -f v2_10_1_ingclass.yaml ``` ## Create Update Strategy diff --git a/docs/examples/echo_server.md b/docs/examples/echo_server.md index 125873085..0ddb322f1 100644 --- a/docs/examples/echo_server.md +++ b/docs/examples/echo_server.md @@ -87,9 +87,9 @@ In this walkthrough, you'll 1. Deploy all the echoserver resources (namespace, service, deployment) ```bash - kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/examples/echoservice/echoserver-namespace.yaml &&\ - kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/examples/echoservice/echoserver-service.yaml &&\ - kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/examples/echoservice/echoserver-deployment.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/examples/echoservice/echoserver-namespace.yaml &&\ + kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/examples/echoservice/echoserver-service.yaml &&\ + kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/examples/echoservice/echoserver-deployment.yaml ``` 1. List all the resources to ensure they were created. @@ -113,7 +113,7 @@ In this walkthrough, you'll 1. Download the echoserver ingress manifest locally. ```bash - wget https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/examples/echoservice/echoserver-ingress.yaml + wget https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/examples/echoservice/echoserver-ingress.yaml ``` 1. Configure the subnets, either by add annotation to the ingress or add tags to subnets. This step is optional in lieu of auto-discovery. @@ -300,7 +300,7 @@ You should get back a valid response. follow below steps if you want to use kube2iam to provide the AWS credentials 1. configure the proper policy - The policy to be used can be fetched from https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.0/docs/install/iam_policy.json + The policy to be used can be fetched from https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/install/iam_policy.json 1. configure the proper role and create the trust relationship You have to find which role is associated with your K8S nodes. Once you found take note of the full arn: diff --git a/helm/aws-load-balancer-controller/Chart.yaml b/helm/aws-load-balancer-controller/Chart.yaml index 1100e7efc..4646868e1 100644 --- a/helm/aws-load-balancer-controller/Chart.yaml +++ b/helm/aws-load-balancer-controller/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: aws-load-balancer-controller description: AWS Load Balancer Controller Helm chart for Kubernetes -version: 1.10.0 -appVersion: v2.10.0 +version: 1.10.1 +appVersion: v2.10.1 home: https://github.com/aws/eks-charts icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png sources: diff --git a/helm/aws-load-balancer-controller/test.yaml b/helm/aws-load-balancer-controller/test.yaml index 42c7421fd..9aa8dfc7f 100644 --- a/helm/aws-load-balancer-controller/test.yaml +++ b/helm/aws-load-balancer-controller/test.yaml @@ -6,7 +6,7 @@ replicaCount: 2 image: repository: public.ecr.aws/eks/aws-load-balancer-controller - tag: v2.10.0 + tag: v2.10.1 pullPolicy: IfNotPresent imagePullSecrets: [] diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index a33542b8b..da6a525a2 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -8,7 +8,7 @@ revisionHistoryLimit: 10 image: repository: public.ecr.aws/eks/aws-load-balancer-controller - tag: v2.10.0 + tag: v2.10.1 pullPolicy: IfNotPresent runtimeClassName: "" diff --git a/version-stable.txt b/version-stable.txt index 10c2c0c3d..8bbb6e406 100644 --- a/version-stable.txt +++ b/version-stable.txt @@ -1 +1 @@ -2.10.0 +2.10.1 diff --git a/version.txt b/version.txt index 10c2c0c3d..8bbb6e406 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -2.10.0 +2.10.1