fix(controller): stuck in scale event after canary service switch delay. Fixes #3412

rollout/canary.go

@@ -185,8 +185,15 @@
  annotationedRSs := int32(0)
  for _, targetRS := range oldRSs {
  if c.rollout.Spec.Strategy.Canary.TrafficRouting != nil && c.isReplicaSetReferenced(targetRS) {
- // We might get here if user interrupted an an update in order to move back to stable.
+ // We might get here if user interrupted an update in order to move back to stable or if the controller could not switch the canary service to the new RS.
  c.log.Infof("Skip scale down of older RS '%s': still referenced", targetRS.Name)
+
+ // If the replicaset is still referenced, the controller should scale it
+ _, desiredRSReplicaCount := replicasetutil.CalculateReplicaCountsForTrafficRoutedCanary(c.rollout, c.rollout.Status.Canary.Weights)
+ _, _, err = c.scaleReplicaSetAndRecordEvent(targetRS, desiredRSReplicaCount)
+ if err != nil {
+ c.log.Errorf("Failed to scale old RS '%s': %v", targetRS.Name, err)
+ }
  continue
  }
  if maxScaleDown <= 0 {

test/e2e/canary_test.go

@@ -222,6 +222,24 @@ spec:
  ExpectCanaryStablePodCount(2, 4)
 }
 
+func (s *CanarySuite) TestRolloutScalingWithServiceSwitchDelay() {
+ s.Given().
+ RolloutObjects(`@functional/alb-canary-rollout-with-delay.yaml`).
+ When().
+ ApplyManifests().
+ WaitForRolloutStatus("Healthy").
+ UpdateSpec().
+ WaitForRolloutStatus("Paused").
+ UpdateSpec().
+ Sleep(1*time.Second).
+ ScaleRollout(3).
+ WaitForRolloutStatus("Paused").
+ Then().
+ ExpectRevisionPodCount("1", 3).
+ ExpectRevisionPodCount("2", 0).
+ ExpectRevisionPodCount("3", 1)
+}
+
 // TestReduceWeightAndHonorMaxUnavailable verifies we honor maxUnavailable when decreasing weight or aborting
 func (s *CanarySuite) TestReduceWeightAndHonorMaxUnavailable() {
  s.Given().

test/e2e/functional/alb-canary-rollout-with-delay.yaml

@@ -0,0 +1,104 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: alb-canary-with-delay-root
+spec:
+ type: NodePort
+ ports:
+ - port: 80
+ targetPort: http
+ protocol: TCP
+ name: http
+ selector:
+ app: alb-canary-with-delay
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: alb-canary-with-delay-desired
+spec:
+ type: NodePort
+ ports:
+ - port: 80
+ targetPort: http
+ protocol: TCP
+ name: http
+ selector:
+ app: alb-canary-with-delay
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: alb-canary-with-delay-stable
+spec:
+ type: NodePort
+ ports:
+ - port: 80
+ targetPort: http
+ protocol: TCP
+ name: http
+ selector:
+ app: alb-canary-with-delay
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: alb-canary-with-delay-ingress
+ annotations:
+ kubernetes.io/ingress.class: alb
+spec:
+ rules:
+ - http:
+ paths:
+ - path: /*
+ pathType: ImplementationSpecific
+ backend:
+ service:
+ name: alb-canary-with-delay-root
+ port:
+ name: use-annotation
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+ name: alb-canary-with-delay
+spec:
+ selector:
+ matchLabels:
+ app: alb-canary-with-delay
+ template:
+ metadata:
+ labels:
+ app: alb-canary-with-delay
+ spec:
+ containers:
+ - name: alb-canary-with-delay
+ lifecycle:
+ postStart:
+ exec:
+ command:
+ - sleep
+ - "2"
+ image: nginx:1.19-alpine
+ ports:
+ - name: http
+ containerPort: 80
+ protocol: TCP
+ resources:
+ requests:
+ memory: 16Mi
+ cpu: 5m
+ strategy:
+ canary:
+ canaryService: alb-canary-with-delay-desired
+ stableService: alb-canary-with-delay-stable
+ trafficRouting:
+ alb:
+ ingress: alb-canary-with-delay-ingress
+ rootService: alb-canary-with-delay-root
+ servicePort: 80
+ steps:
+ - setWeight: 10
+ - pause: {duration: 5s}
+ - setWeight: 20
+ - pause: {duration: 5s}