From 13538cbe9bec2c019013631695237faacc116393 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 21 Nov 2024 08:02:48 -0800 Subject: [PATCH] FEAT: LBC metric collector (#3941) --- main.go | 13 +++- pkg/aws/cloud.go | 13 ++-- pkg/{aws/metrics => metrics/aws}/collector.go | 21 +++---- .../metrics => metrics/aws}/collector_test.go | 2 +- .../metrics => metrics/aws}/instruments.go | 34 +++------- pkg/metrics/lbc/collector.go | 39 ++++++++++++ pkg/metrics/lbc/instruments.go | 38 ++++++++++++ pkg/metrics/lbc/mockcollector.go | 37 +++++++++++ pkg/targetgroupbinding/resource_manager.go | 23 ++++--- .../resource_manager_test.go | 62 ++++++++++++------- 10 files changed, 205 insertions(+), 77 deletions(-) rename pkg/{aws/metrics => metrics/aws}/collector.go (92%) rename pkg/{aws/metrics => metrics/aws}/collector_test.go (98%) rename pkg/{aws/metrics => metrics/aws}/instruments.go (77%) create mode 100644 pkg/metrics/lbc/collector.go create mode 100644 pkg/metrics/lbc/instruments.go create mode 100644 pkg/metrics/lbc/mockcollector.go diff --git a/main.go b/main.go index b6484f24fd..de2a3edc9d 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,8 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/inject" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + awsmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" "sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding" @@ -81,7 +83,14 @@ func main() { ctrl.SetLogger(appLogger) klog.SetLoggerWithOptions(appLogger, klog.ContextualLogger(true)) - cloud, err := aws.NewCloud(controllerCFG.AWSConfig, metrics.Registry, ctrl.Log, nil) + var awsMetricsCollector *awsmetrics.Collector + lbcMetricsCollector := lbcmetrics.NewCollector(metrics.Registry) + + if metrics.Registry != nil { + awsMetricsCollector = awsmetrics.NewCollector(metrics.Registry) + } + + cloud, err := aws.NewCloud(controllerCFG.AWSConfig, awsMetricsCollector, ctrl.Log, nil) if err != nil { setupLog.Error(err, "unable to initialize AWS cloud") os.Exit(1) @@ -113,7 +122,7 @@ func main() { subnetResolver := networking.NewDefaultSubnetsResolver(azInfoProvider, cloud.EC2(), cloud.VpcID(), controllerCFG.ClusterName, ctrl.Log.WithName("subnets-resolver")) multiClusterManager := targetgroupbinding.NewMultiClusterManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log) tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.EC2(), - podInfoRepo, sgManager, sgReconciler, vpcInfoProvider, multiClusterManager, + podInfoRepo, sgManager, sgReconciler, vpcInfoProvider, multiClusterManager, lbcMetricsCollector, cloud.VpcID(), controllerCFG.ClusterName, controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules, controllerCFG.ServiceTargetENISGTags, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log) backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup, diff --git a/pkg/aws/cloud.go b/pkg/aws/cloud.go index cc679b9436..41070e70db 100644 --- a/pkg/aws/cloud.go +++ b/pkg/aws/cloud.go @@ -11,7 +11,6 @@ import ( smithymiddleware "github.com/aws/smithy-go/middleware" "net" "os" - "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/metrics" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle" "sigs.k8s.io/aws-load-balancer-controller/pkg/version" "strings" @@ -21,11 +20,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/go-logr/logr" "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" amerrors "k8s.io/apimachinery/pkg/util/errors" epresolver "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/endpoints" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/provider" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + aws_metrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws" ) const userAgent = "elbv2.k8s.aws" @@ -60,7 +59,7 @@ type Cloud interface { } // NewCloud constructs new Cloud implementation. -func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (Cloud, error) { +func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (Cloud, error) { hasIPv4 := true addrs, err := net.InterfaceAddrs() if err == nil { @@ -122,12 +121,8 @@ func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger l }) } - if metricsRegisterer != nil { - metricsCollector, err := metrics.NewCollector(metricsRegisterer) - if err != nil { - return nil, errors.Wrapf(err, "failed to initialize sdk metrics collector") - } - awsConfig.APIOptions = metrics.WithSDKMetricCollector(metricsCollector, awsConfig.APIOptions) + if metricsCollector != nil { + awsConfig.APIOptions = aws_metrics.WithSDKMetricCollector(metricsCollector, awsConfig.APIOptions) } if awsClientsProvider == nil { diff --git a/pkg/aws/metrics/collector.go b/pkg/metrics/aws/collector.go similarity index 92% rename from pkg/aws/metrics/collector.go rename to pkg/metrics/aws/collector.go index 607d5f3054..f6fe66e980 100644 --- a/pkg/aws/metrics/collector.go +++ b/pkg/metrics/aws/collector.go @@ -1,4 +1,4 @@ -package metrics +package aws import ( "context" @@ -18,24 +18,21 @@ const ( sdkMiddlewareCollectAPIRequestMetric = "collectAPIRequestMetric" ) -type collector struct { +type Collector struct { instruments *instruments } -func NewCollector(registerer prometheus.Registerer) (*collector, error) { - instruments, err := newInstruments(registerer) - if err != nil { - return nil, err - } - return &collector{ +func NewCollector(registerer prometheus.Registerer) *Collector { + instruments := newInstruments(registerer) + return &Collector{ instruments: instruments, - }, nil + } } /* WithSDKMetricCollector is a function that collects prometheus metrics for the AWS SDK Go v2 API calls ad requests */ -func WithSDKMetricCollector(c *collector, apiOptions []func(*smithymiddleware.Stack) error) []func(*smithymiddleware.Stack) error { +func WithSDKMetricCollector(c *Collector, apiOptions []func(*smithymiddleware.Stack) error) []func(*smithymiddleware.Stack) error { apiOptions = append(apiOptions, func(stack *smithymiddleware.Stack) error { return WithSDKCallMetricCollector(c)(stack) }, func(stack *smithymiddleware.Stack) error { @@ -48,7 +45,7 @@ func WithSDKMetricCollector(c *collector, apiOptions []func(*smithymiddleware.St WithSDKCallMetricCollector is a middleware for the AWS SDK Go v2 that collects and reports metrics on API calls. The call metrics are collected after the call is completed */ -func WithSDKCallMetricCollector(c *collector) func(stack *smithymiddleware.Stack) error { +func WithSDKCallMetricCollector(c *Collector) func(stack *smithymiddleware.Stack) error { return func(stack *smithymiddleware.Stack) error { return stack.Initialize.Add(smithymiddleware.InitializeMiddlewareFunc(sdkMiddlewareCollectAPICallMetric, func( ctx context.Context, input smithymiddleware.InitializeInput, next smithymiddleware.InitializeHandler, @@ -91,7 +88,7 @@ func WithSDKCallMetricCollector(c *collector) func(stack *smithymiddleware.Stack WithSDKRequestMetricCollector is a middleware for the AWS SDK Go v2 that collects and reports metrics on API requests. The request metrics are collected after each retry attempts */ -func WithSDKRequestMetricCollector(c *collector) func(stack *smithymiddleware.Stack) error { +func WithSDKRequestMetricCollector(c *Collector) func(stack *smithymiddleware.Stack) error { return func(stack *smithymiddleware.Stack) error { return stack.Finalize.Add(smithymiddleware.FinalizeMiddlewareFunc(sdkMiddlewareCollectAPIRequestMetric, func( ctx context.Context, input smithymiddleware.FinalizeInput, next smithymiddleware.FinalizeHandler, diff --git a/pkg/aws/metrics/collector_test.go b/pkg/metrics/aws/collector_test.go similarity index 98% rename from pkg/aws/metrics/collector_test.go rename to pkg/metrics/aws/collector_test.go index db2579381f..3e52a2dda9 100644 --- a/pkg/aws/metrics/collector_test.go +++ b/pkg/metrics/aws/collector_test.go @@ -1,4 +1,4 @@ -package metrics +package aws import ( "errors" diff --git a/pkg/aws/metrics/instruments.go b/pkg/metrics/aws/instruments.go similarity index 77% rename from pkg/aws/metrics/instruments.go rename to pkg/metrics/aws/instruments.go index e3ca812044..150bf11019 100644 --- a/pkg/aws/metrics/instruments.go +++ b/pkg/metrics/aws/instruments.go @@ -1,11 +1,11 @@ -package metrics +package aws import ( "github.com/prometheus/client_golang/prometheus" ) const ( - metricSubsystemAWS = "aws" + metricSubSystem = "aws" metricAPICallsTotal = "api_calls_total" metricAPICallDurationSeconds = "api_call_duration_seconds" @@ -31,55 +31,41 @@ type instruments struct { } // newInstruments allocates and register new metrics to registerer -func newInstruments(registerer prometheus.Registerer) (*instruments, error) { +func newInstruments(registerer prometheus.Registerer) *instruments { apiCallsTotal := prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPICallsTotal, Help: "Total number of SDK API calls from the customer's code to AWS services", }, []string{labelService, labelOperation, labelStatusCode, labelErrorCode}) apiCallDurationSeconds := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPICallDurationSeconds, Help: "Perceived latency from when your code makes an SDK call, includes retries", }, []string{labelService, labelOperation}) apiCallRetries := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPICallRetries, Help: "Number of times the SDK retried requests to AWS services for SDK API calls", Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, }, []string{labelService, labelOperation}) apiRequestsTotal := prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPIRequestsTotal, Help: "Total number of HTTP requests that the SDK made", }, []string{labelService, labelOperation, labelStatusCode, labelErrorCode}) apiRequestDurationSecond := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: metricSubsystemAWS, + Subsystem: metricSubSystem, Name: metricAPIRequestDurationSeconds, Help: "Latency of an individual HTTP request to the service endpoint", }, []string{labelService, labelOperation}) - if err := registerer.Register(apiCallsTotal); err != nil { - return nil, err - } - if err := registerer.Register(apiCallDurationSeconds); err != nil { - return nil, err - } - if err := registerer.Register(apiCallRetries); err != nil { - return nil, err - } - if err := registerer.Register(apiRequestsTotal); err != nil { - return nil, err - } - if err := registerer.Register(apiRequestDurationSecond); err != nil { - return nil, err - } + registerer.MustRegister(apiCallsTotal, apiCallDurationSeconds, apiCallRetries, apiRequestsTotal, apiRequestDurationSecond) return &instruments{ apiCallsTotal: apiCallsTotal, apiCallDurationSeconds: apiCallDurationSeconds, apiCallRetries: apiCallRetries, apiRequestsTotal: apiRequestsTotal, apiRequestDurationSecond: apiRequestDurationSecond, - }, nil + } } diff --git a/pkg/metrics/lbc/collector.go b/pkg/metrics/lbc/collector.go new file mode 100644 index 0000000000..34da128486 --- /dev/null +++ b/pkg/metrics/lbc/collector.go @@ -0,0 +1,39 @@ +package lbc + +import ( + "github.com/prometheus/client_golang/prometheus" + "time" +) + +type MetricCollector interface { + // ObservePodReadinessGateReady this metric is useful to determine how fast pods are becoming ready in the load balancer. + // Due to some architectural constraints, we can only emit this metric for pods that are using readiness gates. + ObservePodReadinessGateReady(namespace string, tgbName string, duration time.Duration) +} + +type collector struct { + instruments *instruments +} + +type noOpCollector struct{} + +func (n *noOpCollector) ObservePodReadinessGateReady(_ string, _ string, _ time.Duration) { +} + +func NewCollector(registerer prometheus.Registerer) MetricCollector { + if registerer == nil { + return &noOpCollector{} + } + + instruments := newInstruments(registerer) + return &collector{ + instruments: instruments, + } +} + +func (c *collector) ObservePodReadinessGateReady(namespace string, tgbName string, duration time.Duration) { + c.instruments.podReadinessFlipSeconds.With(prometheus.Labels{ + labelNamespace: namespace, + labelName: tgbName, + }).Observe(duration.Seconds()) +} diff --git a/pkg/metrics/lbc/instruments.go b/pkg/metrics/lbc/instruments.go new file mode 100644 index 0000000000..0a38b7f771 --- /dev/null +++ b/pkg/metrics/lbc/instruments.go @@ -0,0 +1,38 @@ +package lbc + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + metricSubsystem = "awslbc" +) + +// These metrics are exported to be used in unit test validation. +const ( + // MetricPodReadinessGateReady tracks the time to flip a readiness gate to true + MetricPodReadinessGateReady = "readiness_gate_ready_seconds" +) + +const ( + labelNamespace = "namespace" + labelName = "name" +) + +type instruments struct { + podReadinessFlipSeconds *prometheus.HistogramVec +} + +// newInstruments allocates and register new metrics to registerer +func newInstruments(registerer prometheus.Registerer) *instruments { + podReadinessFlipSeconds := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Subsystem: metricSubsystem, + Name: MetricPodReadinessGateReady, + Help: "Latency from pod getting added to the load balancer until the readiness gate is flipped to healthy.", + }, []string{labelNamespace, labelName}) + + registerer.MustRegister(podReadinessFlipSeconds) + return &instruments{ + podReadinessFlipSeconds: podReadinessFlipSeconds, + } +} diff --git a/pkg/metrics/lbc/mockcollector.go b/pkg/metrics/lbc/mockcollector.go new file mode 100644 index 0000000000..9c8fb6a43a --- /dev/null +++ b/pkg/metrics/lbc/mockcollector.go @@ -0,0 +1,37 @@ +package lbc + +import ( + "time" +) + +type MockCollector struct { + Invocations map[string][]interface{} +} + +type MockHistogramMetric struct { + namespace string + name string + duration time.Duration +} + +func (m *MockCollector) ObservePodReadinessGateReady(namespace string, tgbName string, d time.Duration) { + m.recordHistogram(MetricPodReadinessGateReady, namespace, tgbName, d) +} + +func (m *MockCollector) recordHistogram(metricName string, namespace string, name string, d time.Duration) { + m.Invocations[metricName] = append(m.Invocations[MetricPodReadinessGateReady], MockHistogramMetric{ + namespace: namespace, + name: name, + duration: d, + }) +} + +func NewMockCollector() MetricCollector { + + mockInvocations := make(map[string][]interface{}) + mockInvocations[MetricPodReadinessGateReady] = make([]interface{}, 0) + + return &MockCollector{ + Invocations: mockInvocations, + } +} diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 9056c36664..af25a824f6 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -6,6 +6,7 @@ import ( elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/aws/smithy-go" "net/netip" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" "time" "k8s.io/client-go/tools/record" @@ -37,7 +38,7 @@ type ResourceManager interface { // NewDefaultResourceManager constructs new defaultResourceManager. func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELBV2, ec2Client services.EC2, podInfoRepo k8s.PodInfoRepo, sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler, - vpcInfoProvider networking.VPCInfoProvider, multiClusterManager MultiClusterManager, + vpcInfoProvider networking.VPCInfoProvider, multiClusterManager MultiClusterManager, metricsCollector lbcmetrics.MetricCollector, vpcID string, clusterName string, failOpenEnabled bool, endpointSliceEnabled bool, disabledRestrictedSGRulesFlag bool, endpointSGTags map[string]string, eventRecorder record.EventRecorder, logger logr.Logger) *defaultResourceManager { @@ -60,6 +61,7 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB vpcInfoProvider: vpcInfoProvider, podInfoRepo: podInfoRepo, multiClusterManager: multiClusterManager, + metricsCollector: metricsCollector, requeueDuration: defaultRequeueDuration, } @@ -78,6 +80,7 @@ type defaultResourceManager struct { vpcInfoProvider networking.VPCInfoProvider podInfoRepo k8s.PodInfoRepo multiClusterManager MultiClusterManager + metricsCollector lbcmetrics.MetricCollector vpcID string requeueDuration time.Duration @@ -240,7 +243,7 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, return "", "", false, err } - anyPodNeedFurtherProbe, err := m.updateTargetHealthPodCondition(ctx, targetHealthCondType, matchedEndpointAndTargets, unmatchedEndpoints) + anyPodNeedFurtherProbe, err := m.updateTargetHealthPodCondition(ctx, targetHealthCondType, matchedEndpointAndTargets, unmatchedEndpoints, tgb) if err != nil { return "", "", false, err } @@ -374,13 +377,13 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a // updateTargetHealthPodCondition will updates pod's targetHealth condition for matchedEndpointAndTargets and unmatchedEndpoints. // returns whether further probe is needed or not func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Context, targetHealthCondType corev1.PodConditionType, - matchedEndpointAndTargets []podEndpointAndTargetPair, unmatchedEndpoints []backend.PodEndpoint) (bool, error) { + matchedEndpointAndTargets []podEndpointAndTargetPair, unmatchedEndpoints []backend.PodEndpoint, tgb *elbv2api.TargetGroupBinding) (bool, error) { anyPodNeedFurtherProbe := false for _, endpointAndTarget := range matchedEndpointAndTargets { pod := endpointAndTarget.endpoint.Pod targetHealth := endpointAndTarget.target.TargetHealth - needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType, tgb) if err != nil { return false, err } @@ -396,7 +399,7 @@ func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Cont Reason: elbv2types.TargetHealthReasonEnumRegistrationInProgress, Description: awssdk.String("Target registration is in progress"), } - needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType, tgb) if err != nil { return false, err } @@ -410,7 +413,7 @@ func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Cont // updateTargetHealthPodConditionForPod updates pod's targetHealth condition for a single pod and its matched target. // returns whether further probe is needed or not. func (m *defaultResourceManager) updateTargetHealthPodConditionForPod(ctx context.Context, pod k8s.PodInfo, - targetHealth *elbv2types.TargetHealth, targetHealthCondType corev1.PodConditionType) (bool, error) { + targetHealth *elbv2types.TargetHealth, targetHealthCondType corev1.PodConditionType, tgb *elbv2api.TargetGroupBinding) (bool, error) { if !pod.HasAnyOfReadinessGates([]corev1.PodConditionType{targetHealthCondType}) { return false, nil } @@ -468,6 +471,12 @@ func (m *defaultResourceManager) updateTargetHealthPodConditionForPod(ctx contex return false, err } + // Only update duration on unhealthy -> healthy flips. + if targetHealthCondStatus == corev1.ConditionTrue && hasExistingTargetHealthCond && !existingTargetHealthCond.LastTransitionTime.IsZero() && existingTargetHealthCond.Status != corev1.ConditionTrue { + delta := newTargetHealthCond.LastTransitionTime.Sub(existingTargetHealthCond.LastTransitionTime.Time) + m.metricsCollector.ObservePodReadinessGateReady(tgb.Namespace, tgb.Name, delta) + } + return needFurtherProbe, nil } @@ -509,7 +518,7 @@ func (m *defaultResourceManager) updatePodAsHealthyForDeletedTGB(ctx context.Con State: elbv2types.TargetHealthStateEnumHealthy, Description: awssdk.String("Target Group Binding is deleted"), } - _, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + _, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType, tgb) if err != nil { return err } diff --git a/pkg/targetgroupbinding/resource_manager_test.go b/pkg/targetgroupbinding/resource_manager_test.go index 1031c1e530..4839b2fb24 100644 --- a/pkg/targetgroupbinding/resource_manager_test.go +++ b/pkg/targetgroupbinding/resource_manager_test.go @@ -3,6 +3,8 @@ package targetgroupbinding import ( "context" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" "testing" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -26,6 +28,9 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing pods []*corev1.Pod } + tgbName := "tgb" + tgbNamespace := "tgbNamespace" + type args struct { pod k8s.PodInfo targetHealth *elbv2types.TargetHealth @@ -33,12 +38,13 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing } tests := []struct { - name string - env env - args args - want bool - wantPod *corev1.Pod - wantErr error + name string + env env + args args + want bool + wantPod *corev1.Pod + wantMetric bool + wantErr error }{ { name: "pod contains readinessGate and targetHealth is healthy - add pod condition", @@ -137,10 +143,11 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing Status: corev1.PodStatus{ Conditions: []corev1.PodCondition{ { - Type: "target-health.elbv2.k8s.aws/my-tgb", - Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), - Reason: "Elb.RegistrationInProgress", - Status: corev1.ConditionFalse, + Type: "target-health.elbv2.k8s.aws/my-tgb", + Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), + Reason: "Elb.RegistrationInProgress", + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), }, { Type: corev1.ContainersReady, @@ -162,10 +169,11 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing }, Conditions: []corev1.PodCondition{ { - Type: "target-health.elbv2.k8s.aws/my-tgb", - Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), - Reason: "Elb.RegistrationInProgress", - Status: corev1.ConditionFalse, + Type: "target-health.elbv2.k8s.aws/my-tgb", + Message: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), + Reason: "Elb.RegistrationInProgress", + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), }, { Type: corev1.ContainersReady, @@ -178,7 +186,8 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing }, targetHealthCondType: "target-health.elbv2.k8s.aws/my-tgb", }, - want: false, + want: false, + wantMetric: true, wantPod: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -226,10 +235,11 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing Status: corev1.PodStatus{ Conditions: []corev1.PodCondition{ { - Type: "target-health.elbv2.k8s.aws/my-tgb", - Status: corev1.ConditionFalse, - Reason: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), - Message: "Target registration is in progress", + Type: "target-health.elbv2.k8s.aws/my-tgb", + Status: corev1.ConditionFalse, + Reason: string(elbv2types.TargetHealthReasonEnumRegistrationInProgress), + Message: "Target registration is in progress", + LastTransitionTime: metav1.Now(), }, { Type: corev1.ContainersReady, @@ -382,10 +392,15 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing k8sClient := testclient.NewClientBuilder().WithScheme(k8sSchema).Build() m := &defaultResourceManager{ - k8sClient: k8sClient, - logger: logr.New(&log.NullLogSink{}), + k8sClient: k8sClient, + logger: logr.New(&log.NullLogSink{}), + metricsCollector: lbcmetrics.NewMockCollector(), } + tgb := &elbv2api.TargetGroupBinding{} + tgb.Name = tgbName + tgb.Namespace = tgbNamespace + ctx := context.Background() for _, pod := range tt.env.pods { err := k8sClient.Create(ctx, pod.DeepCopy()) @@ -393,7 +408,7 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing } got, err := m.updateTargetHealthPodConditionForPod(context.Background(), - tt.args.pod, tt.args.targetHealth, tt.args.targetHealthCondType) + tt.args.pod, tt.args.targetHealth, tt.args.targetHealthCondType, tgb) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -410,6 +425,9 @@ func Test_defaultResourceManager_updateTargetHealthPodConditionForPod(t *testing } assert.True(t, cmp.Equal(tt.wantPod, updatedPod, opts), "diff", cmp.Diff(tt.wantPod, updatedPod, opts)) } + + mockCollector := m.metricsCollector.(*lbcmetrics.MockCollector) + assert.Equal(t, tt.wantMetric, len(mockCollector.Invocations[lbcmetrics.MetricPodReadinessGateReady]) == 1) }) } }