MM-62023: Reboot DB instances if needed (#859)

* Reboot DB instances if needed

If the database contains pending-reboot parameters, we need to manually
reboot all instances for those parameters to be applied.

* Apply review comments

- Wrap error when calling HasPendingRebootDBParams
- Use a single channel containing a struct with both the bool and error
  in HasPendingRebootDBParams
- Move all new DB-related methods to terraform/db_operations.go

deployment/terraform/create.go

@@ -342,6 +342,15 @@ func (t *Terraform) PostProcessDatabase(extAgent *ssh.ExtAgent) error {
 		}
 	}
 
+	needsReboot, err := t.HasPendingRebootDBParams()
+	if err != nil {
+		return fmt.Errorf("failed to check whether the DB has pending-reboot parameters: %w", err)
+	}
+
+	if needsReboot {
+		return t.RebootDBInstances(extAgent)
+	}
+
 	return nil
 }
 

deployment/terraform/db_operations.go

@@ -9,7 +9,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sync"
 	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/rds"
+	"github.com/mattermost/mattermost-load-test-ng/deployment/terraform/ssh"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
 )
 
 // StopDB stops the DB cluster and syncs the changes.
@@ -114,3 +120,158 @@ func (t *Terraform) DBStatus() (string, error) {
 
 	return out.DBCluster[0].Status, nil
 }
+
+// HasPendingRebootDBParams queries the deployed DB cluster and checks whether
+// there is at least a DB instance whose status is "pending-reboot"
+func (t *Terraform) HasPendingRebootDBParams() (bool, error) {
+	// Build the RDS client
+	cfg, err := t.GetAWSConfig()
+	if err != nil {
+		return false, fmt.Errorf("failed to get AWS config: %w", err)
+	}
+	rdsClient := rds.NewFromConfig(cfg)
+
+	// Check in parallel whether each DB instance needs to be rebooted
+	type retValue struct {
+		needsReboot bool
+		err         error
+	}
+	retChan := make(chan retValue, len(t.output.DBCluster.Instances))
+	var wg sync.WaitGroup
+	for _, instance := range t.output.DBCluster.Instances {
+		wg.Add(1)
+		go func(dbId string) {
+			defer wg.Done()
+			needsReboot, err := hasPendingRebootDBParams(rdsClient, dbId)
+			retChan <- retValue{needsReboot, err}
+		}(instance.DBIdentifier)
+	}
+
+	wg.Wait()
+	close(retChan)
+
+	needsReboot := false
+	var finalErr error
+	for b := range retChan {
+		needsReboot = needsReboot || b.needsReboot
+		finalErr = errors.Join(finalErr, b.err)
+	}
+
+	return needsReboot, finalErr
+}
+
+// hasPendingRebootDBParams queries the specified DB instance and checks whether
+// its status is "pending-reboot"
+func hasPendingRebootDBParams(rdsClient *rds.Client, dbId string) (bool, error) {
+	describeParams := &rds.DescribeDBInstancesInput{
+		DBInstanceIdentifier: model.NewPointer(dbId),
+	}
+	describeOut, err := rdsClient.DescribeDBInstances(context.Background(), describeParams)
+	if err != nil {
+		return false, fmt.Errorf("error describing DB instance %q: %w", dbId, err)
+	}
+
+	if len(describeOut.DBInstances) < 1 {
+		return false, fmt.Errorf("describe instances returned no instances")
+	}
+
+	for _, group := range describeOut.DBInstances[0].DBParameterGroups {
+		if group.ParameterApplyStatus == nil {
+			return false, fmt.Errorf("parameter group has no ParameterApplyStatus")
+		}
+
+		if *group.ParameterApplyStatus == "pending-reboot" {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// RebootDBInstances reboots all deployed database instances, blocking the call
+// until the status of each of them is back to "available"
+func (t *Terraform) RebootDBInstances(extAgent *ssh.ExtAgent) error {
+	// Build the RDS client
+	cfg, err := t.GetAWSConfig()
+	if err != nil {
+		return fmt.Errorf("failed to get AWS config: %w", err)
+	}
+	rdsClient := rds.NewFromConfig(cfg)
+
+	// Reboot each DB instance in parallel
+	errChan := make(chan error, len(t.output.DBCluster.Instances))
+	var wg sync.WaitGroup
+	for _, instance := range t.output.DBCluster.Instances {
+		wg.Add(1)
+		go func(dbId string) {
+			defer wg.Done()
+			errChan <- rebootDBInstance(rdsClient, dbId)
+		}(instance.DBIdentifier)
+	}
+
+	wg.Wait()
+	close(errChan)
+
+	var finalErr error
+	for err := range errChan {
+		finalErr = errors.Join(finalErr, err)
+	}
+
+	return finalErr
+}
+
+// rebootDBInstance reboots the specified database instance, blocking the call
+// until its status is back to "available"
+func rebootDBInstance(rdsClient *rds.Client, dbId string) error {
+	params := &rds.RebootDBInstanceInput{
+		DBInstanceIdentifier: model.NewPointer(dbId),
+	}
+
+	out, err := rdsClient.RebootDBInstance(context.Background(), params)
+	if err != nil {
+		return fmt.Errorf("failed to reboot DB instance: %w", err)
+	}
+
+	mlog.Info("DB instance reboot has started",
+		mlog.String("id", dbId),
+		mlog.String("status", *out.DBInstance.DBInstanceStatus))
+
+	// Wait for the DB instance to become available, or fail after 15 minutes
+	timeout := time.After(15 * time.Minute)
+	for {
+		select {
+		case <-timeout:
+			return fmt.Errorf("timeout reached, instance is not available yet")
+		case <-time.After(30 * time.Second):
+			describeParams := &rds.DescribeDBInstancesInput{
+				DBInstanceIdentifier: model.NewPointer(dbId),
+			}
+			describeOut, err := rdsClient.DescribeDBInstances(context.Background(), describeParams)
+			if err != nil {
+				return fmt.Errorf("error describing DB instance %q: %w", dbId, err)
+			}
+
+			if len(describeOut.DBInstances) < 1 {
+				return fmt.Errorf("describe instances returned no instances")
+			}
+
+			if describeOut.DBInstances[0].DBInstanceStatus == nil {
+				return fmt.Errorf("describe instances returned no status")
+			}
+
+			status := *describeOut.DBInstances[0].DBInstanceStatus
+
+			// Finish when the DB is completely rebooted
+			if status == "available" {
+				mlog.Info("DB instance is now available.",
+					mlog.String("id", dbId),
+					mlog.String("status", status))
+				return nil
+			}
+
+			mlog.Info("DB instance is not available yet. Waiting 30 seconds...",
+				mlog.String("id", dbId),
+				mlog.String("status", status))
+		}
+	}
+}

deployment/terraform/output.go

@@ -21,6 +21,7 @@ type output struct {
 			Endpoint          string `json:"endpoint"`
 			ClusterIdentifier string `json:"cluster_identifier"`
 			Writer            bool   `json:"writer"`
+			DBIdentifier      string `json:"identifier"`
 		} `json:"value"`
 	} `json:"dbCluster"`
 	Agents struct {
@@ -112,8 +113,9 @@ type Tags struct {
 
 // DBInstance defines an RDS instance resource.
 type DBInstance struct {
-	Endpoint string
-	IsWriter bool
+	DBIdentifier string
+	Endpoint     string
+	IsWriter     bool
 }
 
 // DBCluster defines a RDS cluster instance resource.
@@ -169,8 +171,9 @@ func (t *Terraform) loadOutput() error {
 	if len(o.DBCluster.Value) > 0 {
 		for _, inst := range o.DBCluster.Value {
 			outputv2.DBCluster.Instances = append(outputv2.DBCluster.Instances, DBInstance{
-				Endpoint: inst.Endpoint,
-				IsWriter: inst.Writer,
+				DBIdentifier: inst.DBIdentifier,
+				Endpoint:     inst.Endpoint,
+				IsWriter:     inst.Writer,
 			})
 		}
 		outputv2.DBCluster.ClusterIdentifier = o.DBCluster.Value[0].ClusterIdentifier

go.mod

@@ -24,10 +24,11 @@ require (
 
 require (
 	github.com/Nerzal/gocloak/v13 v13.9.0
-	github.com/aws/aws-sdk-go-v2 v1.31.0
+	github.com/aws/aws-sdk-go-v2 v1.32.5
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
 	github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.37.3
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0
+	github.com/aws/aws-sdk-go-v2/service/rds v1.91.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3
 	github.com/gliderlabs/ssh v0.1.1
 	github.com/grafana/alloy/syntax v0.1.0
@@ -60,15 +61,15 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
-	github.com/aws/smithy-go v1.21.0 // indirect
+	github.com/aws/smithy-go v1.22.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -94,6 +95,7 @@ require (
 	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/imkira/go-interpol v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect

go.sum

@@ -33,8 +33,8 @@ github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNY
 github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
 github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk=
 github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U=
-github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA=
+github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo=
+github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc=
 github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
@@ -43,10 +43,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVO
 github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk=
@@ -55,14 +55,16 @@ github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.37.3 h1:pnvujeesw3tP0iDLK
 github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.37.3/go.mod h1:eJZGfJNuTmvBgiy2O5XIPlHMBi4GUYoJoKZ6U6wCVVk=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0 h1:ta62lid9JkIpKZtZZXSj6rP2AqY5x1qYGq53ffxqD9Q=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0/go.mod h1:o6QDjdVKpP5EF0dp/VlvqckzuSDATr1rLdHt3A5m0YY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 h1:QFASJGfT8wMXtuP3D5CRmMjARHv9ZmzFUMJznHDOY3w=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5/go.mod h1:QdZ3OmoIjSX+8D1OPAzPxDfjXASbBMDsz9qvtyIhtik=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 h1:Xbwbmk44URTiHNx6PNo0ujDE6ERlsCKJD3u1zfnzAPg=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20/go.mod h1:oAfOFzUB14ltPZj1rWwRc3d/6OgD76R8KlvU3EqM9Fg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg=
+github.com/aws/aws-sdk-go-v2/service/rds v1.91.0 h1:eqHz3Uih+gb0vLE5Cc4Xf733vOxsxDp6GFUUVQU4d7w=
+github.com/aws/aws-sdk-go-v2/service/rds v1.91.0/go.mod h1:h2jc7IleH3xHY7y+h8FH7WAZcz3IVLOB6/jXotIQ/qU=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3 h1:3zt8qqznMuAZWDTDpcwv9Xr11M/lVj2FsRR7oYBt0OA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q=
 github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
@@ -71,8 +73,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrA
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
 github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
 github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
-github.com/aws/smithy-go v1.21.0 h1:H7L8dtDRk0P1Qm6y0ji7MCYMQObJ5R9CRpyPhRUkLYA=
-github.com/aws/smithy-go v1.21.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
+github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -229,6 +231,10 @@ github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6Pyu
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c=
 github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
@@ -687,6 +693,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=