Allow scaling events to be logged during cluster validation

cloudmock/gce/mockcompute/instance_group_manager.go

@@ -125,6 +125,10 @@ func (c *instanceGroupManagerClient) List(ctx context.Context, project, zone str
  return l, nil
 }
 
+func (c *instanceGroupManagerClient) ListErrors(ctx context.Context, project, zone, name string) ([]*compute.InstanceManagedByIgmError, error) {
+ return []*compute.InstanceManagedByIgmError{}, nil
+}
+
 func (c *instanceGroupManagerClient) ListManagedInstances(ctx context.Context, project, zone, name string) ([]*compute.ManagedInstance, error) {
  var instances []*compute.ManagedInstance
  return instances, nil

pkg/cloudinstances/cloud_instance_group.go

@@ -19,6 +19,7 @@ package cloudinstances
 import (
  "fmt"
  "strings"
+ "time"
 
  v1 "k8s.io/api/core/v1"
  "k8s.io/klog/v2"
@@ -35,11 +36,21 @@ type CloudInstanceGroup struct {
  MinSize int
  TargetSize int
  MaxSize int
+ Events []ScalingEvent
 
  // Raw allows for the implementer to attach an object, for tracking additional state
  Raw interface{}
 }
 
+type ScalingEvent struct {
+ Timestamp time.Time
+ Description string
+}
+
+func (e ScalingEvent) String() string {
+ return fmt.Sprintf("%s: %s", e.Timestamp.Format(time.RFC3339), e.Description)
+}
+
 // NewCloudInstance creates a new CloudInstance
 func (c *CloudInstanceGroup) NewCloudInstance(instanceId string, status string, node *v1.Node) (*CloudInstance, error) {
  if instanceId == "" {

pkg/validation/validate_cluster.go

@@ -301,6 +301,16 @@ func (v *ValidationCluster) validateNodes(cloudGroups map[string]*cloudinstances
  cloudGroup.TargetSize),
  InstanceGroup: cloudGroup.InstanceGroup,
  })
+ if klog.V(4).Enabled() {
+ // Log scaling events
+ klog.V(4).Infof("Scaling events for instance group: %v", cloudGroup.InstanceGroup.Name)
+ sort.Slice(cloudGroup.Events, func(i, j int) bool {
+ return cloudGroup.Events[i].Timestamp.Before(cloudGroup.Events[j].Timestamp)
+ })
+ for _, event := range cloudGroup.Events {
+ klog.Infof("%v", event)
+ }
+ }
  }
 
  for _, member := range allMembers {

tests/e2e/scenarios/upgrade-ab/run-test.sh

@@ -129,10 +129,9 @@ sleep 120s
 
 ${CHANNELS} apply channel "$KOPS_STATE_STORE"/"${CLUSTER_NAME}"/addons/bootstrap-channel.yaml --yes -v4
 
-"${KOPS_B}" rolling-update cluster
-"${KOPS_B}" rolling-update cluster --yes --validation-timeout 30m
+"${KOPS_B}" rolling-update cluster --yes --validation-timeout 30m -v 4
 
-"${KOPS_B}" validate cluster
+"${KOPS_B}" validate cluster -v 4
 
 # Verify kubeconfig-a still works
 kubectl get nodes -owide --kubeconfig="${KUBECONFIG_A}"

upup/pkg/fi/cloudup/awsup/aws_cloud.go

@@ -1196,13 +1196,29 @@ func awsBuildCloudInstanceGroup(ctx context.Context, c AWSCloud, cluster *kops.C
  return nil, fmt.Errorf("failed to fetch instances: %v", err)
  }
 
+ scalingReq := &autoscaling.DescribeScalingActivitiesInput{
+ AutoScalingGroupName: g.AutoScalingGroupName,
+ }
+ scalingEvents := make([]cloudinstances.ScalingEvent, 0)
+ c.Autoscaling().DescribeScalingActivitiesPagesWithContext(ctx, scalingReq, func(p *autoscaling.DescribeScalingActivitiesOutput, lastPage bool) bool {
+ for _, activity := range p.Activities {
+ event := cloudinstances.ScalingEvent{
+ Timestamp: aws.TimeValue(activity.StartTime),
+ Description: aws.StringValue(activity.Description),
+ }
+ scalingEvents = append(scalingEvents, event)
+ }
+ return true
+ })
+
  cg := &cloudinstances.CloudInstanceGroup{
  HumanName: aws.StringValue(g.AutoScalingGroupName),
  InstanceGroup: ig,
  MinSize: int(aws.Int64Value(g.MinSize)),
  TargetSize: int(aws.Int64Value(g.DesiredCapacity)),
  MaxSize: int(aws.Int64Value(g.MaxSize)),
  Raw: g,
+ Events: scalingEvents,
  }
 
  for _, i := range g.Instances {

upup/pkg/fi/cloudup/gce/compute.go

@@ -673,6 +673,7 @@ type InstanceGroupManagerClient interface {
  Delete(project, zone, name string) (*compute.Operation, error)
  Get(project, zone, name string) (*compute.InstanceGroupManager, error)
  List(ctx context.Context, project, zone string) ([]*compute.InstanceGroupManager, error)
+ ListErrors(ctx context.Context, project, zone, name string) ([]*compute.InstanceManagedByIgmError, error)
  ListManagedInstances(ctx context.Context, project, zone, name string) ([]*compute.ManagedInstance, error)
  RecreateInstances(project, zone, name, id string) (*compute.Operation, error)
  SetTargetPools(project, zone, name string, targetPools []string) (*compute.Operation, error)
@@ -709,6 +710,17 @@ func (c *instanceGroupManagerClientImpl) List(ctx context.Context, project, zone
  return ms, nil
 }
 
+func (c *instanceGroupManagerClientImpl) ListErrors(ctx context.Context, project, zone, name string) ([]*compute.InstanceManagedByIgmError, error) {
+ var igmErrors []*compute.InstanceManagedByIgmError
+ if err := c.srv.ListErrors(project, zone, name).Pages(ctx, func(page *compute.InstanceGroupManagersListErrorsResponse) error {
+ igmErrors = append(igmErrors, page.Items...)
+ return nil
+ }); err != nil {
+ return nil, err
+ }
+ return igmErrors, nil
+}
+
 func (c *instanceGroupManagerClientImpl) ListManagedInstances(ctx context.Context, project, zone, name string) ([]*compute.ManagedInstance, error) {
  var instances []*compute.ManagedInstance
  if err := c.srv.ListManagedInstances(project, zone, name).Pages(ctx, func(page *compute.InstanceGroupManagersListManagedInstancesResponse) error {

upup/pkg/fi/cloudup/gce/instancegroups.go

@@ -22,6 +22,7 @@ import (
  "fmt"
  "hash/fnv"
  "strings"
+ "time"
 
  compute "google.golang.org/api/compute/v1"
  v1 "k8s.io/api/core/v1"
@@ -153,13 +154,22 @@ func getCloudGroups(c GCECloud, cluster *kops.Cluster, instancegroups []*kops.In
  continue
  }
 
+ igmErrors, err := c.Compute().InstanceGroupManagers().ListErrors(ctx, project, zoneName, name)
+ if err != nil {
+ return nil, fmt.Errorf("getting MIG Errors: %v", err)
+ }
+ events, err := igmErrorsToScalingEvents(igmErrors)
+ if err != nil {
+ return nil, fmt.Errorf("error converting MIG errors to scaling events: %v", err)
+ }
  g := &cloudinstances.CloudInstanceGroup{
  HumanName: mig.Name,
  InstanceGroup: ig,
  MinSize: int(mig.TargetSize),
  TargetSize: int(mig.TargetSize),
  MaxSize: int(mig.TargetSize),
  Raw: mig,
+ Events: events,
  }
  groups[mig.Name] = g
 
@@ -264,6 +274,24 @@ func matchInstanceGroup(mig *compute.InstanceGroupManager, c *kops.Cluster, inst
  return matches[0], nil
 }
 
+func igmErrorsToScalingEvents(igmErrors []*compute.InstanceManagedByIgmError) ([]cloudinstances.ScalingEvent, error) {
+ events := make([]cloudinstances.ScalingEvent, 0)
+ for _, error := range igmErrors {
+ ts, err := time.Parse(time.RFC3339, error.Timestamp)
+ if err != nil {
+ return nil, err
+ }
+ if error.Error != nil {
+ activity := cloudinstances.ScalingEvent{
+ Timestamp: ts,
+ Description: error.Error.Message,
+ }
+ events = append(events, activity)
+ }
+ }
+ return events, nil
+}
+
 func addCloudInstanceData(cm *cloudinstances.CloudInstance, instance *compute.Instance) {
  cm.MachineType = LastComponent(instance.MachineType)
  cm.Status = instance.Status