From cc98cdd263b52ade4fa3f06e922d22f5c17f9d63 Mon Sep 17 00:00:00 2001 From: Andre Leite Date: Tue, 4 Feb 2025 18:42:31 -0300 Subject: [PATCH 1/2] CLD-8818:Modify e2e to fetch AMI from aws --- e2e/tests/shared/shared.go | 29 +++++++++++++++++++++++ internal/mocks/aws-tools/client.go | 8 +++++++ internal/supervisor/installation_test.go | 4 ++++ internal/tools/aws/client.go | 1 + internal/tools/aws/ec2.go | 30 ++++++++++++++++++++++++ model/kops_metadata.go | 2 ++ 6 files changed, 74 insertions(+) diff --git a/e2e/tests/shared/shared.go b/e2e/tests/shared/shared.go index 467f89580..084e22df9 100644 --- a/e2e/tests/shared/shared.go +++ b/e2e/tests/shared/shared.go @@ -22,6 +22,7 @@ import ( "github.com/mattermost/mattermost-cloud/e2e/pkg/eventstest" "github.com/mattermost/mattermost-cloud/e2e/tests/state" "github.com/mattermost/mattermost-cloud/e2e/workflow" + awsTools "github.com/mattermost/mattermost-cloud/internal/tools/aws" "github.com/mattermost/mattermost-cloud/model" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -256,16 +257,44 @@ func fetchAMI(cloudClient *model.Client, logger logrus.FieldLogger) (string, err if err != nil { return "", errors.Wrap(err, "failed to get clusters to fetch AMI") } + if len(clusters) == 0 { return "", errors.Errorf("no clusters found to fetch AMI") } + if clusters[0].Provider == model.ProviderExternal { + ami, err := fetchAMIFromAWS(logger) + if err != nil { + return "", errors.Wrap(err, "failed to fetch AMI from AWS") + } + logrus.Infof("Fetched AMI from AWS: %q", ami) + return ami, nil + } + ami := clusters[0].ProvisionerMetadataKops.AMI logrus.Infof("Fetched AMI from existing cluster: %q", ami) return ami, nil } +func fetchAMIFromAWS(logger logrus.FieldLogger) (string, error) { + awsConfig, err := awsTools.NewAWSConfig(context.TODO()) + if err != nil { + return "", errors.Wrap(err, "failed to build aws configuration") + } + + awsClient, err := awsTools.NewAWSClientWithConfig(&awsConfig, logrus.New()) + if err != nil { + return "", errors.Wrap(err, "failed to build AWS client") + } + + ami, err := awsClient.GetAMIByTag("Name", model.AMDKopsAmiName, logger) + if err != nil { + return "", errors.Wrap(err, "failed to get AMI image by tag") + } + return ami, nil +} + func TestMain(m *testing.M) { // This is mainly used to send a notification when tests are finished to a mattermost webhook // provided with the WEBHOOOK_URL environment variable. diff --git a/internal/mocks/aws-tools/client.go b/internal/mocks/aws-tools/client.go index 39ba0539b..a764d8afd 100644 --- a/internal/mocks/aws-tools/client.go +++ b/internal/mocks/aws-tools/client.go @@ -265,6 +265,14 @@ func (m *MockAWS) IsValidAMI(AMIImage string, logger logrus.FieldLogger) (bool, return ret0, ret1 } +func (m *MockAWS) GetAMIByTag(tagKey, tagValue string, logger logrus.FieldLogger) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAMIByTag", tagKey, tagValue, logger) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + // IsValidAMI indicates an expected call of IsValidAMI func (mr *MockAWSMockRecorder) IsValidAMI(AMIImage, logger interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() diff --git a/internal/supervisor/installation_test.go b/internal/supervisor/installation_test.go index 2513b3a80..1f2b92387 100644 --- a/internal/supervisor/installation_test.go +++ b/internal/supervisor/installation_test.go @@ -593,6 +593,10 @@ func (a *mockAWS) IsValidAMI(AMIID string, logger log.FieldLogger) (bool, error) return true, nil } +func (a *mockAWS) GetAMIByTag(tagKey, tagValue string, logger log.FieldLogger) (string, error) { + return "ami-1234567890", nil +} + func (a *mockAWS) GeneratePerseusUtilitySecret(clusterID string, logger log.FieldLogger) (*corev1.Secret, error) { return nil, nil } diff --git a/internal/tools/aws/client.go b/internal/tools/aws/client.go index e71872c89..edc6f2165 100644 --- a/internal/tools/aws/client.go +++ b/internal/tools/aws/client.go @@ -52,6 +52,7 @@ type AWS interface { UpsertPublicCNAMEs(dnsNames []string, endpoints []string, logger log.FieldLogger) error IsValidAMI(AMIImage string, logger log.FieldLogger) (bool, error) + GetAMIByTag(tagKey, tagValue string, logger log.FieldLogger) (string, error) S3EnsureBucketDeleted(bucketName string, logger log.FieldLogger) error S3EnsureObjectDeleted(bucketName, path string) error diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index 01d7e66b1..32476bc1d 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "sort" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -339,3 +340,32 @@ set -o xtrace /etc/eks/bootstrap.sh '%s' --apiserver-endpoint '%s' --b64-cluster-ca '%s' --use-max-pods false --kubelet-extra-args '--max-pods=%d'` return fmt.Sprintf(dataTemplate, *eksCluster.Name, *eksCluster.Endpoint, *eksCluster.CertificateAuthority.Data, data.MaxPodsPerNode) } + +func (a *Client) GetAMIByTag(tagKey, tagValue string, logger log.FieldLogger) (string, error) { + ctx := context.TODO() + + images, err := a.Service().ec2.DescribeImages(ctx, &ec2.DescribeImagesInput{ + Owners: []string{"self"}, + Filters: []ec2Types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", tagKey)), + Values: []string{tagValue}, + }, + }, + }) + if err != nil { + return "", errors.Wrap(err, "failed to describe images by tag") + } + + if len(images.Images) == 0 { + a.logger.Info("No images found matching the criteria.") + return "", nil + } + + //Sort images by creation date + sort.Slice(images.Images, func(i, j int) bool { + return *images.Images[i].CreationDate > *images.Images[j].CreationDate + }) + + return *images.Images[0].ImageId, nil +} diff --git a/model/kops_metadata.go b/model/kops_metadata.go index eee9366a0..717f9449f 100644 --- a/model/kops_metadata.go +++ b/model/kops_metadata.go @@ -15,6 +15,8 @@ import ( const ( ProvisionerKops = "kops" + AMDKopsAmiName = "mattermost-cloud-kops-hardened-*-amd64" + ARMKopsAmiName = "mattermost-cloud-kops-hardened-*-arm64" ) // KopsMetadata is the provisioner metadata stored in a model.Cluster. From 97f138ab4c70af95a2b32b4ad2019504837a2e98 Mon Sep 17 00:00:00 2001 From: Andre Leite Date: Tue, 4 Feb 2025 23:25:43 -0300 Subject: [PATCH 2/2] CLD-8818:Make mocks --- internal/mocks/aws-tools/client.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/mocks/aws-tools/client.go b/internal/mocks/aws-tools/client.go index a764d8afd..0e470969a 100644 --- a/internal/mocks/aws-tools/client.go +++ b/internal/mocks/aws-tools/client.go @@ -265,6 +265,13 @@ func (m *MockAWS) IsValidAMI(AMIImage string, logger logrus.FieldLogger) (bool, return ret0, ret1 } +// IsValidAMI indicates an expected call of IsValidAMI +func (mr *MockAWSMockRecorder) IsValidAMI(AMIImage, logger interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsValidAMI", reflect.TypeOf((*MockAWS)(nil).IsValidAMI), AMIImage, logger) +} + +// GetAMIByTag mocks base method func (m *MockAWS) GetAMIByTag(tagKey, tagValue string, logger logrus.FieldLogger) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAMIByTag", tagKey, tagValue, logger) @@ -273,10 +280,10 @@ func (m *MockAWS) GetAMIByTag(tagKey, tagValue string, logger logrus.FieldLogger return ret0, ret1 } -// IsValidAMI indicates an expected call of IsValidAMI -func (mr *MockAWSMockRecorder) IsValidAMI(AMIImage, logger interface{}) *gomock.Call { +// GetAMIByTag indicates an expected call of GetAMIByTag +func (mr *MockAWSMockRecorder) GetAMIByTag(tagKey, tagValue, logger interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsValidAMI", reflect.TypeOf((*MockAWS)(nil).IsValidAMI), AMIImage, logger) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAMIByTag", reflect.TypeOf((*MockAWS)(nil).GetAMIByTag), tagKey, tagValue, logger) } // S3EnsureBucketDeleted mocks base method