From 17967de4effd744d3a7111d964d013eddd969867 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 4 Apr 2023 15:01:48 +0200 Subject: [PATCH 01/29] provisioner flag --- cmd/cloud/cluster.go | 9 ++- cmd/cloud/cluster_flag.go | 9 ++- cmd/cloud/server_flag.go | 24 ++++++ internal/provisioner/crossplane/eks_types.go | 62 +++++++++++++++ model/crossplane_metadata.go | 9 +++ xrd/main.go | 81 ++++++++++++++++++++ xrd/xrd.yaml | 33 ++++++++ 7 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 internal/provisioner/crossplane/eks_types.go create mode 100644 model/crossplane_metadata.go create mode 100644 xrd/main.go create mode 100644 xrd/xrd.yaml diff --git a/cmd/cloud/cluster.go b/cmd/cloud/cluster.go index b34692e2c..622b01549 100644 --- a/cmd/cloud/cluster.go +++ b/cmd/cloud/cluster.go @@ -105,6 +105,7 @@ func executeClusterCreateCmd(flags clusterCreateFlags) error { request := &model.CreateClusterRequest{ Provider: flags.provider, + Provisioner: flags.provisioner, Version: flags.version, AMI: flags.ami, Zones: strings.Split(flags.zones, ","), @@ -113,11 +114,9 @@ func executeClusterCreateCmd(flags clusterCreateFlags) error { Annotations: flags.annotations, Networking: flags.networking, VPC: flags.vpc, - Provisioner: model.ProvisionerKops, } - if flags.useEKS { - request.Provisioner = model.ProvisionerEKS + if flags.provisioner == model.ProvisionerEKS { request.ClusterRoleARN = flags.clusterRoleARN request.NodeRoleARN = flags.nodeRoleARN request.NodeGroupWithPublicSubnet = flags.nodegroupWithPublicSubnet @@ -538,6 +537,10 @@ func defaultClustersTableData(clusters []*model.ClusterDTO) ([]string, [][]strin provisionerMetadata = cluster.ProvisionerMetadataKops.GetCommonMetadata() masterCount = cluster.ProvisionerMetadataKops.MasterCount masterInstanceType = cluster.ProvisionerMetadataKops.MasterInstanceType + } else if cluster.Provisioner == model.ProvisionerCrossplane && cluster.ProvisionerMetadataCrossplane != nil { + provisionerMetadata = cluster.ProvisionerMetadataCrossplane.GetCommonMetadata() + masterCount = cluster.ProvisionerMetadataCrossplane.NodeCount + masterInstanceType = cluster.ProvisionerMetadataCrossplane.InstanceType } else if cluster.Provisioner == model.ProvisionerEKS && cluster.ProvisionerMetadataEKS != nil { provisionerMetadata = cluster.ProvisionerMetadataEKS.GetCommonMetadata() masterCount = 1 diff --git a/cmd/cloud/cluster_flag.go b/cmd/cloud/cluster_flag.go index 2be3abaed..d0b657cac 100644 --- a/cmd/cloud/cluster_flag.go +++ b/cmd/cloud/cluster_flag.go @@ -1,3 +1,7 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + package main import ( @@ -21,6 +25,7 @@ func (flags *clusterFlags) addFlags(command *cobra.Command) { type createRequestOptions struct { provider string + provisioner string version string ami string zones string @@ -30,7 +35,6 @@ type createRequestOptions struct { vpc string clusterRoleARN string nodeRoleARN string - useEKS bool additionalNodeGroups map[string]string nodegroupWithPublicSubnet []string } @@ -45,9 +49,10 @@ func (flags *createRequestOptions) addFlags(command *cobra.Command) { command.Flags().StringVar(&flags.networking, "networking", "calico", "Networking mode to use, for example: weave, calico, canal, amazon-vpc-routed-eni") command.Flags().StringVar(&flags.vpc, "vpc", "", "Set to use a shared VPC") + command.Flags().StringVar(&flags.provisioner, "provisioner", "kops", "Provisioner to create cluster with.") + command.Flags().StringVar(&flags.clusterRoleARN, "cluster-role-arn", "", "AWS role ARN for cluster.") command.Flags().StringVar(&flags.nodeRoleARN, "node-role-arn", "", "AWS role ARN for node.") - command.Flags().BoolVar(&flags.useEKS, "eks", false, "Create EKS cluster.") command.Flags().StringToStringVar(&flags.additionalNodeGroups, "additional-node-groups", nil, "Additional nodegroups to create. The key is the name of the nodegroup and the value is the size constant.") command.Flags().StringSliceVar(&flags.nodegroupWithPublicSubnet, "nodegroup-with-public-subnet", nil, "Nodegroups to create with public subnet. The value is the name of the nodegroup.") } diff --git a/cmd/cloud/server_flag.go b/cmd/cloud/server_flag.go index 957d4f05e..2b09ce6a9 100644 --- a/cmd/cloud/server_flag.go +++ b/cmd/cloud/server_flag.go @@ -8,6 +8,7 @@ import ( "time" toolsAWS "github.com/mattermost/mattermost-cloud/internal/tools/aws" + "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -176,6 +177,26 @@ func (flags *serverFlagChanged) addFlags(command *cobra.Command) { flags.isKeepFileStoreDataChanged = command.Flags().Changed("keep-filestore-data") } +type crossplaneOptions struct { + enableCrossplane bool + k8sUseInClusterConfig bool + k8sKubeconfigPath string +} + +func (flags *crossplaneOptions) addFlags(command *cobra.Command) { + command.Flags().BoolVar(&flags.enableCrossplane, "enable-crossplane", false, "Whether to use Crossplane to provision resources.") + command.Flags().BoolVar(&flags.k8sUseInClusterConfig, "k8s-use-in-cluster-config", false, "Whether to use in-cluster config for k8s client.") + command.Flags().StringVar(&flags.k8sKubeconfigPath, "k8s-kubeconfig-path", "", "Path to kubeconfig file for k8s client.") + command.MarkFlagsMutuallyExclusive("k8s-use-in-cluster-config", "k8s-kubeconfig-path") +} + +func (flags *crossplaneOptions) valid(command *cobra.Command) error { + if flags.enableCrossplane && !flags.k8sUseInClusterConfig && flags.k8sKubeconfigPath == "" { + return errors.New("k8s-kubeconfig-path or k8s-use-in-cluster-config is required when enable-crossplane is set to true") + } + return nil +} + type serverFlags struct { supervisorOptions schedulingOptions @@ -185,6 +206,8 @@ type serverFlags struct { dbUtilizationSettings serverFlagChanged + crossplaneOptions + listen string metricsPort int @@ -211,6 +234,7 @@ func (flags *serverFlags) addFlags(command *cobra.Command) { flags.pgBouncerConfig.addFlags(command) flags.installationOptions.addFlags(command) flags.dbUtilizationSettings.addFlags(command) + flags.crossplaneOptions.addFlags(command) command.Flags().StringVar(&flags.listen, "listen", ":8075", "The interface and port on which to listen.") command.Flags().IntVar(&flags.metricsPort, "metrics-port", 8076, "Port on which the metrics server should be listening.") diff --git a/internal/provisioner/crossplane/eks_types.go b/internal/provisioner/crossplane/eks_types.go new file mode 100644 index 000000000..13cac6787 --- /dev/null +++ b/internal/provisioner/crossplane/eks_types.go @@ -0,0 +1,62 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + +package crossplane + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + GroupVersion = schema.GroupVersion{Group: "cloud.mattermost.io", Version: "v1alpha1"} +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type EKS struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec EKSSpec `json:"spec"` +} + +type EKSMetadata struct { + Name string `json:"name"` +} + +type EKSSpec struct { + CompositionSelector EKSSpecCompositionSelector `json:"compositionSelector"` + ID string `json:"id"` + Parameters EKSSpecParameters `json:"parameters"` + ResourceConfig EKSSpecResourceConfig `json:"resourceConfig"` +} + +type EKSSpecCompositionSelector struct { + MatchLabels EKSSpecCompositionSelectorMatchLabels `json:"matchLabels"` +} + +type EKSSpecCompositionSelectorMatchLabels struct { + Provider string `json:"provider"` + Service string `json:"service"` +} + +type EKSSpecParameters struct { + Version string `json:"version"` + AccountID string `json:"account_id"` + Region string `json:"region"` + Environment string `json:"environment"` + ClusterShortName string `json:"cluster_short_name"` + EndpointPrivateAccess bool `json:"endpoint_private_access"` + EndpointPublicAccess bool `json:"endpoint_public_access"` + VpcID string `json:"vpc_id"` + SubnetIds []string `json:"subnet_ids"` + PrivateSubnetIds []string `json:"private_subnet_ids"` + NodeCount int `json:"node_count"` + InstanceType string `json:"instance_type"` + ImageID string `json:"image_id"` + LaunchTemplateVersion string `json:"launch_template_version"` +} + +type EKSSpecResourceConfig struct { + ProviderConfigName string `json:"providerConfigName"` +} diff --git a/model/crossplane_metadata.go b/model/crossplane_metadata.go new file mode 100644 index 000000000..30e487f98 --- /dev/null +++ b/model/crossplane_metadata.go @@ -0,0 +1,9 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + +package model + +const ( + ProvisionerCrossplane = "crossplane" +) diff --git a/xrd/main.go b/xrd/main.go new file mode 100644 index 000000000..298b8605c --- /dev/null +++ b/xrd/main.go @@ -0,0 +1,81 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + +package main + +import ( + "encoding/json" + + "github.com/mattermost/mattermost-cloud/internal/provisioner/crossplane" + "github.com/mattermost/mattermost-cloud/k8s" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func main() { + // ctx := context.TODO() + logger := logrus.New() + namespace := "testing" + obj := crossplane.EKS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: namespace, + // Labels: map[string]string{"test": "test"}, + // Annotations: , + }, + Spec: crossplane.EKSSpec{}, + } + + // obj := corev1.Pod{ + // // TypeMeta: metav1.TypeMeta{ + // // Kind: "Pod", + // // APIVersion: "v1", + // // }, + // ObjectMeta: metav1.ObjectMeta{ + // Name: "test-nginx", + // Namespace: namespace, + // }, + // Spec: corev1.PodSpec{ + // Containers: []corev1.Container{{ + // Image: "nginx", + // }}, + // }, + // } + + // cfg, err := rest.InClusterConfig() + // if err != nil { + // panic(err) + // } + + // client, err := k8s.NewFromConfig(cfg, logger) + // if err != nil { + // panic(err) + // } + + objBytes, err := json.Marshal(obj) + if err != nil { + panic(err) + } + + client, err := k8s.NewFromFile("/Users/fmartingr/.kube/config_kind_provisioner", logger) + if err != nil { + panic(err) + } + + req := client.Clientset.CoreV1().RESTClient(). + Post(). + Resource("cloud.mattermost.sio/mmK8ss"). + Namespace(obj.Namespace). + Name(obj.Name). + Body(objBytes) + + output, execErr := client.RemoteCommand("POST", req.URL()) + + if execErr != nil { + logger.WithError(execErr).Error("failed") + } else { + logger.Debugf(string(output)) + } + +} diff --git a/xrd/xrd.yaml b/xrd/xrd.yaml new file mode 100644 index 000000000..684b4dc34 --- /dev/null +++ b/xrd/xrd.yaml @@ -0,0 +1,33 @@ +apiVersion: cloud.mattermost.io/v1alpha1 +kind: mmK8s +metadata: + name: b07c8248b874b3821af37d81d3 +spec: + compositionSelector: + matchLabels: + provider: aws + service: eks + id: b07c8248b874b3821af37d81d3 + parameters: + version: "1.23" + account_id: "926412419614" + region: us-east-1 + environment: dev + cluster_short_name: crossplane-test + endpoint_private_access: true + endpoint_public_access: false + vpc_id: vpc-09d44077df9934f96 + subnet_ids: + - "subnet-0866bf0625a68488a" + - "subnet-0f74e2dfcdcdc18b2" + - "subnet-0e123b8281481294a" + - "subnet-0ccc5bf96d05d2a3a" + private_subnet_ids: + - "subnet-0e123b8281481294a" + - "subnet-0ccc5bf96d05d2a3a" + node_count: 1 + instance_type: t3.large + image_id: ami-0bc89720a2e35652e + launch_template_version: "2" + resourceConfig: + providerConfigName: crossplane-provider-config From c7617c7f736eb0d963efbc8353217c81de214ff1 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 4 Apr 2023 15:02:16 +0200 Subject: [PATCH 02/29] crossplane provisioner --- cmd/cloud/server.go | 37 ++++++++++++++++++++++++++++- internal/provisioner/provisioner.go | 15 ++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cmd/cloud/server.go b/cmd/cloud/server.go index 32798336d..0991566cb 100644 --- a/cmd/cloud/server.go +++ b/cmd/cloud/server.go @@ -34,6 +34,7 @@ import ( "github.com/mattermost/mattermost-cloud/internal/tools/kops" "github.com/mattermost/mattermost-cloud/internal/tools/terraform" "github.com/mattermost/mattermost-cloud/internal/tools/utils" + "github.com/mattermost/mattermost-cloud/k8s" "github.com/mattermost/mattermost-cloud/model" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -55,6 +56,12 @@ func newCmdServer() *cobra.Command { }, PreRun: func(command *cobra.Command, args []string) { flags.serverFlagChanged.addFlags(command) // To populate flag change variables. + if err := flags.crossplaneOptions.valid(command); err != nil { + _ = command.Usage() + logger.Error(err) + os.Exit(1) + } + deprecationWarnings(logger, command) if flags.enableLogStacktrace { @@ -235,6 +242,9 @@ func executeServerCmd(flags serverFlags) error { "disable-db-init-check": flags.disableDBInitCheck, "enable-route53": flags.enableRoute53, "disable-dns-updates": flags.disableDNSUpdates, + "enable-crossplane": flags.enableCrossplane, + "k8s-use-incluster-config": flags.k8sUseInClusterConfig, + "k8s-kubeconfig-path": flags.k8sKubeconfigPath, }).Info("Starting Mattermost Provisioning Server") // Warn on settings we consider to be non-production. @@ -278,6 +288,31 @@ func executeServerCmd(flags serverFlags) error { EtcdManagerEnv: etcdManagerEnv, } + // Crossplane provisioner configuration. + var crossplaneProvisioner *provisioner.CrossplaneProvisioner + if flags.enableCrossplane { + var k8sClient *k8s.KubeClient + var err error + if flags.k8sUseInClusterConfig { + k8sClient, err = k8s.NewFromInClusterConfig(logger) + if err != nil { + return errors.Wrap(err, "failed to build k8s client") + } + } else if flags.k8sKubeconfigPath != "" { + k8sClient, err = k8s.NewFromFile(flags.k8sKubeconfigPath, logger) + if err != nil { + return errors.Wrap(err, "failed to build k8s client") + } + } + crossplaneProvisioner = provisioner.NewCrossplaneProvisioner( + k8sClient, + awsClient, + provisioningParams, + sqlStore, + logger, + ) + } + resourceUtil := utils.NewResourceUtil(instanceID, awsClient, dbClusterUtilizationSettingsFromFlags(flags), flags.disableDBInitCheck) kopsProvisioner := provisioner.NewKopsProvisioner( @@ -295,7 +330,7 @@ func executeServerCmd(flags serverFlags) error { ) provisionerObj := provisioner.NewProvisioner( - kopsProvisioner, eksProvisioner, + kopsProvisioner, eksProvisioner, crossplaneProvisioner, provisioningParams, awsClient, resourceUtil, diff --git a/internal/provisioner/provisioner.go b/internal/provisioner/provisioner.go index 456d70e41..e0513782c 100644 --- a/internal/provisioner/provisioner.go +++ b/internal/provisioner/provisioner.go @@ -15,8 +15,9 @@ import ( ) type ClusterProvisionerOption struct { - kopsProvisioner *KopsProvisioner - eksProvisioner *EKSProvisioner + kopsProvisioner *KopsProvisioner + eksProvisioner *EKSProvisioner + crossplaneProvisioner *CrossplaneProvisioner } func (c ClusterProvisionerOption) GetClusterProvisioner(provisioner string) supervisor.ClusterProvisioner { @@ -24,6 +25,10 @@ func (c ClusterProvisionerOption) GetClusterProvisioner(provisioner string) supe return c.eksProvisioner } + if provisioner == model.ProvisionerCrossplane { + return c.crossplaneProvisioner + } + return c.kopsProvisioner } @@ -68,6 +73,7 @@ var _ kube = (*KopsProvisioner)(nil) func NewProvisioner( kopsProvisioner *KopsProvisioner, eksProvisioner *EKSProvisioner, + crossplaneProvisioner *CrossplaneProvisioner, params ProvisioningParams, awsClient aws.AWS, resourceUtil *utils.ResourceUtil, @@ -78,8 +84,9 @@ func NewProvisioner( return Provisioner{ ClusterProvisionerOption: ClusterProvisionerOption{ - kopsProvisioner: kopsProvisioner, - eksProvisioner: eksProvisioner, + kopsProvisioner: kopsProvisioner, + eksProvisioner: eksProvisioner, + crossplaneProvisioner: crossplaneProvisioner, }, params: params, awsClient: awsClient, From 4a1b8ef60a65908e0f9be7c319b7dee0ea2dd326 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 4 Apr 2023 15:03:17 +0200 Subject: [PATCH 03/29] basic crossplane provisioner (wip) --- internal/api/cluster.go | 7 + internal/provisioner/crossplane/eks_types.go | 62 ------ .../provisioner/crossplane_provisioner.go | 186 ++++++++++++++++++ k8s/client.go | 20 ++ model/cluster.go | 47 +++-- model/cluster_request.go | 4 +- model/crossplane_metadata.go | 101 ++++++++++ 7 files changed, 348 insertions(+), 79 deletions(-) delete mode 100644 internal/provisioner/crossplane/eks_types.go create mode 100644 internal/provisioner/crossplane_provisioner.go diff --git a/internal/api/cluster.go b/internal/api/cluster.go index 706d0e1cf..6dc911f3d 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -120,6 +120,13 @@ func handleCreateCluster(c *Context, w http.ResponseWriter, r *http.Request) { if createClusterRequest.Provisioner == model.ProvisionerEKS { cluster.ProvisionerMetadataEKS = &model.EKSMetadata{} cluster.ProvisionerMetadataEKS.ApplyClusterCreateRequest(createClusterRequest) + } else if createClusterRequest.Provisioner == model.ProvisionerCrossplane { + cluster.ProvisionerMetadataCrossplane = &model.CrossplaneMetadata{ + // TODO: Defaults to first zone in the AWS metadata for now. + Region: cluster.ProviderMetadataAWS.Zones[0], + } + cluster.ProvisionerMetadataCrossplane.SetDefaults() + cluster.ProvisionerMetadataCrossplane.ApplyClusterCreateRequest(createClusterRequest) } else { cluster.ProvisionerMetadataKops = &model.KopsMetadata{} cluster.ProvisionerMetadataKops.ApplyClusterCreateRequest(createClusterRequest) diff --git a/internal/provisioner/crossplane/eks_types.go b/internal/provisioner/crossplane/eks_types.go deleted file mode 100644 index 13cac6787..000000000 --- a/internal/provisioner/crossplane/eks_types.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. -// - -package crossplane - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -var ( - GroupVersion = schema.GroupVersion{Group: "cloud.mattermost.io", Version: "v1alpha1"} -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -type EKS struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - Spec EKSSpec `json:"spec"` -} - -type EKSMetadata struct { - Name string `json:"name"` -} - -type EKSSpec struct { - CompositionSelector EKSSpecCompositionSelector `json:"compositionSelector"` - ID string `json:"id"` - Parameters EKSSpecParameters `json:"parameters"` - ResourceConfig EKSSpecResourceConfig `json:"resourceConfig"` -} - -type EKSSpecCompositionSelector struct { - MatchLabels EKSSpecCompositionSelectorMatchLabels `json:"matchLabels"` -} - -type EKSSpecCompositionSelectorMatchLabels struct { - Provider string `json:"provider"` - Service string `json:"service"` -} - -type EKSSpecParameters struct { - Version string `json:"version"` - AccountID string `json:"account_id"` - Region string `json:"region"` - Environment string `json:"environment"` - ClusterShortName string `json:"cluster_short_name"` - EndpointPrivateAccess bool `json:"endpoint_private_access"` - EndpointPublicAccess bool `json:"endpoint_public_access"` - VpcID string `json:"vpc_id"` - SubnetIds []string `json:"subnet_ids"` - PrivateSubnetIds []string `json:"private_subnet_ids"` - NodeCount int `json:"node_count"` - InstanceType string `json:"instance_type"` - ImageID string `json:"image_id"` - LaunchTemplateVersion string `json:"launch_template_version"` -} - -type EKSSpecResourceConfig struct { - ProviderConfigName string `json:"providerConfigName"` -} diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go new file mode 100644 index 000000000..d840e7c57 --- /dev/null +++ b/internal/provisioner/crossplane_provisioner.go @@ -0,0 +1,186 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + +package provisioner + +import ( + "context" + "fmt" + + crossplaneV1Alpha1 "github.com/mattermost/mattermost-cloud-crossplane/apis/crossplane/v1alpha1" + "github.com/mattermost/mattermost-cloud/internal/supervisor" + "github.com/mattermost/mattermost-cloud/internal/tools/aws" + "github.com/mattermost/mattermost-cloud/k8s" + "github.com/mattermost/mattermost-cloud/model" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // CrossplaneProvisionerType is provisioner type for EKS clusters. + CrossplaneProvisionerType = "crossplane" + + // crossplaneProvisionerNamespace the namespace where the crossplane resources are created. + crossplaneProvisionerNamespace = "mattermost-cloud" +) + +// CrossplaneProvisioner provisions clusters using Crossplane +type CrossplaneProvisioner struct { + awsClient aws.AWS + kubeClient *k8s.KubeClient + clusterStore clusterUpdateStore + parameters ProvisioningParams + logger log.FieldLogger +} + +var _ supervisor.ClusterProvisioner = (*CrossplaneProvisioner)(nil) + +// NewCrossplaneProvisioner creates a new Crossplane provisioner. +func NewCrossplaneProvisioner( + kubeClient *k8s.KubeClient, + awsClient aws.AWS, + parameters ProvisioningParams, + clusterStore clusterUpdateStore, + logger log.FieldLogger, +) *CrossplaneProvisioner { + return &CrossplaneProvisioner{ + kubeClient: kubeClient, + awsClient: awsClient, + parameters: parameters, + clusterStore: clusterStore, + logger: logger, + } +} + +// PrepareCluster prepares the cluster for provisioning by assigning it a name (if not manually +// provided) and claiming the VPC required for the cluster to be provisioned. +func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) bool { + if cluster.ProvisionerMetadataCrossplane.Name == "" { + cluster.ProvisionerMetadataCrossplane.Name = fmt.Sprintf("%s-crossplane-k8s-local", cluster.ID) + } + + var ( + resources aws.ClusterResources + err error + ) + if cluster.ProvisionerMetadataCrossplane.VPC == "" { + resources, err = provisioner.awsClient.GetAndClaimVpcResources(cluster, provisioner.parameters.Owner, provisioner.logger) + } else { + resources, err = provisioner.awsClient.ClaimVPC(cluster.ProvisionerMetadataCrossplane.VPC, cluster, provisioner.parameters.Owner, provisioner.logger) + } + if err != nil { + provisioner.logger.WithError(err).WithField("vpc", cluster.ProvisionerMetadataCrossplane.VPC).Error("Failed to claim VPC resources") + return false + } + cluster.ProvisionerMetadataCrossplane.VPC = resources.VpcID + cluster.ProvisionerMetadataCrossplane.PublicSubnets = resources.PublicSubnetsIDs + cluster.ProvisionerMetadataCrossplane.PrivateSubnets = resources.PrivateSubnetIDs + + return true +} + +// CreateCluster creates the Crossplane cluster resource in the kubernetes CNC cluster. +func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) error { + ctx := context.TODO() + obj := &crossplaneV1Alpha1.MMK8S{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.ID, + Namespace: crossplaneProvisionerNamespace, + }, + Spec: crossplaneV1Alpha1.EKSSpec{ + ID: cluster.ID, + Parameters: crossplaneV1Alpha1.EKSSpecParameters{ + Version: cluster.ProvisionerMetadataCrossplane.KubernetesVersion, + AccountID: cluster.ProvisionerMetadataCrossplane.AccountID, + Region: cluster.ProvisionerMetadataCrossplane.Region, + Environment: "dev", + ClusterShortName: cluster.ID, + EndpointPrivateAccess: true, + EndpointPublicAccess: true, + VpcID: cluster.ProvisionerMetadataCrossplane.VPC, + SubnetIds: cluster.ProvisionerMetadataCrossplane.PublicSubnets, + PrivateSubnetIds: cluster.ProvisionerMetadataCrossplane.PrivateSubnets, + NodeCount: int(cluster.ProvisionerMetadataCrossplane.NodeCount), + InstanceType: cluster.ProvisionerMetadataCrossplane.InstanceType, + ImageID: cluster.ProvisionerMetadataCrossplane.AMI, + LaunchTemplateVersion: *cluster.ProvisionerMetadataCrossplane.LaunchTemplateVersion, + }, + ResourceConfig: crossplaneV1Alpha1.EKSSpecResourceConfig{}, + }, + } + + _, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Create(ctx, obj, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "error creating object") + } + + return nil +} + +// CheckClusterCreated checks if cluster creation finished. +func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Cluster) (bool, error) { + + if false { + // TODO: check status of the cluster + cluster.State = model.ClusterStateStable + return true, provisioner.clusterStore.UpdateCluster(cluster) + } + + return true, nil +} + +// CreateNodes is no-op. +func (provisioner *CrossplaneProvisioner) CreateNodes(cluster *model.Cluster) error { + return nil +} + +// CheckNodesCreated is no-op. +func (provisioner *CrossplaneProvisioner) CheckNodesCreated(cluster *model.Cluster) (bool, error) { + return true, nil +} + +// ProvisionCluster +func (provisioner *CrossplaneProvisioner) ProvisionCluster(cluster *model.Cluster) error { + return nil +} + +// UpgradeCluster is no-op. +func (provisioner *CrossplaneProvisioner) UpgradeCluster(cluster *model.Cluster) error { + return nil +} + +// RotateClusterNodes is no-op. +func (provisioner *CrossplaneProvisioner) RotateClusterNodes(cluster *model.Cluster) error { + return nil +} + +// ResizeCluster is no-op. +func (provisioner *CrossplaneProvisioner) ResizeCluster(cluster *model.Cluster) error { + return nil +} + +// DeleteCluster deletes Crossplane cluster. +func (provisioner *CrossplaneProvisioner) DeleteCluster(cluster *model.Cluster) (bool, error) { + logger := provisioner.logger.WithField("cluster", cluster.ID) + + err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ID, metav1.DeleteOptions{}) + if err != nil { + return false, errors.Wrap(err, "failed to delete crossplane resource") + } + + err = provisioner.awsClient.ReleaseVpc(cluster, logger) + if err != nil { + return false, errors.Wrap(err, "failed to release cluster VPC") + } + + logger.Info("Successfully deleted Crossplane cluster") + + return true, nil +} + +// RefreshClusterMetadata is no-op. +func (provisioner *CrossplaneProvisioner) RefreshClusterMetadata(cluster *model.Cluster) error { + return nil +} diff --git a/k8s/client.go b/k8s/client.go index 27478715f..48e1da9f9 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -5,8 +5,10 @@ package k8s import ( + "github.com/pkg/errors" log "github.com/sirupsen/logrus" + cloudCrossplaneClientV1alpha1 "github.com/mattermost/mattermost-cloud-crossplane/client/clientset/versioned" mmclientv1alpha1 "github.com/mattermost/mattermost-operator/pkg/client/clientset/versioned" mmclientv1beta1 "github.com/mattermost/mattermost-operator/pkg/client/v1beta1/clientset/versioned" monitoringclientV1 "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" @@ -23,6 +25,7 @@ type KubeClient struct { config *rest.Config Clientset kubernetes.Interface ApixClientset apixclient.Interface + CrossplaneClient cloudCrossplaneClientV1alpha1.Interface MattermostClientsetV1Alpha mmclientv1alpha1.Interface MattermostClientsetV1Beta mmclientv1beta1.Interface MonitoringClientsetV1 monitoringclientV1.Interface @@ -36,6 +39,16 @@ func NewFromConfig(config *rest.Config, logger log.FieldLogger) (*KubeClient, er return createKubeClient(config, logger) } +// NewFromInClusterConfig returns a new KubeClient for accessing the kubernetes API using the +// in-cluster configuration. +func NewFromInClusterConfig(logger log.FieldLogger) (*KubeClient, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get in-cluster config") + } + return createKubeClient(config, logger) +} + // NewFromFile returns a new KubeClient for accessing the kubernetes API. (previously named 'New') func NewFromFile(configLocation string, logger log.FieldLogger) (*KubeClient, error) { config, err := clientcmd.BuildConfigFromFlags("", configLocation) @@ -76,14 +89,21 @@ func createKubeClient(config *rest.Config, logger log.FieldLogger) (*KubeClient, if err != nil { return nil, err } + kubeagClientset, err := kubeagclient.NewForConfig(config) if err != nil { return nil, err } + cloudCrossplaneClient, err := cloudCrossplaneClientV1alpha1.NewForConfig(config) + if err != nil { + return nil, err + } + return &KubeClient{ config: config, Clientset: clientset, + CrossplaneClient: cloudCrossplaneClient, MattermostClientsetV1Alpha: mattermostV1AlphaClientset, MattermostClientsetV1Beta: mattermostV1BetaClientset, MonitoringClientsetV1: monitoringV1Clientset, diff --git a/model/cluster.go b/model/cluster.go index f58e14ad4..30af25c15 100644 --- a/model/cluster.go +++ b/model/cluster.go @@ -8,6 +8,8 @@ import ( "encoding/json" "io" "regexp" + + "github.com/pkg/errors" ) //go:generate provisioner-code-gen generate --out-file=cluster_gen.go --boilerplate-file=../hack/boilerplate/boilerplate.generatego.txt --type=github.com/mattermost/mattermost-cloud/model.Cluster --generator=get_id,get_state,is_deleted,as_resources @@ -23,21 +25,22 @@ const ( // Cluster represents a Kubernetes cluster. type Cluster struct { - ID string - State string - Provider string - ProviderMetadataAWS *AWSMetadata - Provisioner string - ProvisionerMetadataKops *KopsMetadata - ProvisionerMetadataEKS *EKSMetadata - UtilityMetadata *UtilityMetadata - AllowInstallations bool - CreateAt int64 - DeleteAt int64 - APISecurityLock bool - LockAcquiredBy *string - LockAcquiredAt int64 - Networking string + ID string + State string + Provider string + ProviderMetadataAWS *AWSMetadata + Provisioner string + ProvisionerMetadataCrossplane *CrossplaneMetadata + ProvisionerMetadataEKS *EKSMetadata + ProvisionerMetadataKops *KopsMetadata + UtilityMetadata *UtilityMetadata + AllowInstallations bool + CreateAt int64 + DeleteAt int64 + APISecurityLock bool + LockAcquiredBy *string + LockAcquiredAt int64 + Networking string } // Clone returns a deep copy the cluster. @@ -57,6 +60,20 @@ func (c *Cluster) ToDTO(annotations []*Annotation) *ClusterDTO { } } +// GetProviderMetadata returns the provisioner metadata for the cluster. +func (c *Cluster) GetProvisionerMetadata() (any, error) { + switch c.Provisioner { + case ProvisionerCrossplane: + return c.ProvisionerMetadataCrossplane, nil + case ProvisionerEKS: + return c.ProvisionerMetadataEKS, nil + case ProvisionerKops: + return c.ProvisionerMetadataKops, nil + default: + return nil, errors.Errorf("unknown provisioner %s", c.Provisioner) + } +} + // ClusterFromReader decodes a json-encoded cluster from the given io.Reader. func ClusterFromReader(reader io.Reader) (*Cluster, error) { cluster := Cluster{} diff --git a/model/cluster_request.go b/model/cluster_request.go index b29e2f036..f84d50ad7 100644 --- a/model/cluster_request.go +++ b/model/cluster_request.go @@ -22,6 +22,7 @@ const ( // CreateClusterRequest specifies the parameters for a new cluster. type CreateClusterRequest struct { Provider string `json:"provider,omitempty"` + Provisioner string `json:"provisioner,omitempty"` Zones []string `json:"zones,omitempty"` Version string `json:"version,omitempty"` AMI string `json:"ami,omitempty"` @@ -39,7 +40,6 @@ type CreateClusterRequest struct { MaxPodsPerNode int64 `json:"max-pods-per-node,omitempty"` ClusterRoleARN string `json:"cluster-role-arn,omitempty"` NodeRoleARN string `json:"node-role-arn,omitempty"` - Provisioner string `json:"provisioner,omitempty"` AdditionalNodeGroups map[string]NodeGroupMetadata `json:"additional-node-groups,omitempty"` NodeGroupWithPublicSubnet []string `json:"nodegroup-with-public-subnet,omitempty"` } @@ -136,7 +136,7 @@ func (request *CreateClusterRequest) Validate() error { if request.Provider != ProviderAWS { return errors.Errorf("unsupported provider %s", request.Provider) } - if request.Provisioner != ProvisionerKops && request.Provisioner != ProvisionerEKS { + if request.Provisioner != ProvisionerKops && request.Provisioner != ProvisionerEKS && request.Provisioner != ProvisionerCrossplane { return errors.Errorf("unsupported provisioner %s", request.Provisioner) } if !ValidClusterVersion(request.Version) { diff --git a/model/crossplane_metadata.go b/model/crossplane_metadata.go index 30e487f98..96cce8c83 100644 --- a/model/crossplane_metadata.go +++ b/model/crossplane_metadata.go @@ -4,6 +4,107 @@ package model +import ( + "encoding/json" + + "github.com/aws/smithy-go/ptr" +) + const ( ProvisionerCrossplane = "crossplane" + + // defaultLaunchTemplateVersion is the default launch template version to use. + defaultLaunchTemplateVersion = "2" + + // defaultKubernetesVersion is the default Kubernetes version to use. + defaultKubernetesVersion = "latest" + + // defaultInstanceType is the default AWS instance type to use. + defaultInstanceType = "t3.medium" ) + +type CrossplaneMetadata struct { + ChangeRequest *CrossplaneMetadataRequestedState + Name string + AccountID string + AMI string + KubernetesVersion string + LaunchTemplateVersion *string + PrivateSubnets []string + PublicSubnets []string + Region string + VPC string + InstanceType string + NodeCount int64 +} + +func (m *CrossplaneMetadata) ApplyClusterCreateRequest(createRequest *CreateClusterRequest) bool { + m.ChangeRequest = &CrossplaneMetadataRequestedState{ + AMI: createRequest.AMI, + ClusterRoleARN: createRequest.ClusterRoleARN, + MaxPodsPerNode: createRequest.MaxPodsPerNode, + Networking: createRequest.Networking, + Version: createRequest.Version, + VPC: createRequest.VPC, + } + + return true +} + +func (m *CrossplaneMetadata) GetCommonMetadata() ProvisionerMetadata { + return ProvisionerMetadata{ + Name: m.Name, + Version: m.KubernetesVersion, + AMI: m.AMI, + NodeInstanceType: m.InstanceType, + NodeMinCount: m.NodeCount, + NodeMaxCount: m.NodeCount, + MaxPodsPerNode: m.NodeCount, + VPC: m.VPC, + } +} + +func (m *CrossplaneMetadata) SetDefaults() { + if m.LaunchTemplateVersion == nil { + m.LaunchTemplateVersion = ptr.String(defaultLaunchTemplateVersion) + } + + if m.KubernetesVersion == "" { + m.KubernetesVersion = defaultKubernetesVersion + } + + if m.InstanceType == "" { + m.InstanceType = defaultInstanceType + } +} + +// CrossplaneMetadataRequestedState is the requested state for crossplane metadata. +type CrossplaneMetadataRequestedState struct { + AMI string + ClusterRoleARN string + LaunchTemplateVersion *string + MaxPodsPerNode int64 + Networking string + Version string + VPC string +} + +// NewCrossplaneMetadataFromJSON creates an instance of CrossplaneMetadata given the raw database +// provisioner metadata. +func NewCrossplaneMetadataFromJSON(metadataJSON []byte) (*CrossplaneMetadata, error) { + // Check if length of metadata is 0 as opposed to if the value is nil. This + // is done to avoid an issue encountered where the metadata value provided + // had a length of 0, but had non-zero capacity. + if len(metadataJSON) == 0 || string(metadataJSON) == "null" { + // TODO: remove "null" check after sqlite is gone. + return nil, nil + } + + metadata := CrossplaneMetadata{} + err := json.Unmarshal(metadataJSON, &metadata) + if err != nil { + return nil, err + } + + return &metadata, nil +} From ed2bee1ea5b3b23477e53872974240a7057c609e Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 4 Apr 2023 15:03:26 +0200 Subject: [PATCH 04/29] crossplane supervisor (wip) --- internal/supervisor/crossplane.go | 55 +++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 internal/supervisor/crossplane.go diff --git a/internal/supervisor/crossplane.go b/internal/supervisor/crossplane.go new file mode 100644 index 000000000..82cc71c98 --- /dev/null +++ b/internal/supervisor/crossplane.go @@ -0,0 +1,55 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + +package supervisor + +import ( + "github.com/mattermost/mattermost-cloud/internal/metrics" + "github.com/mattermost/mattermost-cloud/model" + log "github.com/sirupsen/logrus" +) + +// ClusterSupervisor finds clusters pending work and effects the required changes. +// +// The degree of parallelism is controlled by a weighted semaphore, intended to be shared with +// other clients needing to coordinate background jobs. +type CrossplaneSupervisor struct { + store clusterStore + provisioner ClusterProvisionerOption + eventsProducer eventProducer + instanceID string + metrics *metrics.CloudMetrics + logger log.FieldLogger +} + +func NewCrossplaneSupervisor( + store clusterStore, + provisioner ClusterProvisionerOption, + eventProducer eventProducer, + instanceID string, + logger log.FieldLogger, + metrics *metrics.CloudMetrics, +) *CrossplaneSupervisor { + return &CrossplaneSupervisor{ + store: store, + provisioner: provisioner, + eventsProducer: eventProducer, + instanceID: instanceID, + metrics: metrics, + logger: logger, + } +} + +// Shutdown performs graceful shutdown tasks for the supervisor. +func (s *CrossplaneSupervisor) Shutdown() { + s.logger.Debug("Shutting down crossplane supervisor") +} + +// Do looks gets the list of clusters and syncs them. +func (s *CrossplaneSupervisor) Do() error { + return nil +} + +// Supervise schedules the required work on the given cluster. +func (s *CrossplaneSupervisor) Supervise(cluster *model.Cluster) {} From 4948fdf726364876509c422cc1c280adc5492431 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 4 Apr 2023 15:03:47 +0200 Subject: [PATCH 05/29] store crossplane provisioner metadata in db --- internal/store/cluster.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/internal/store/cluster.go b/internal/store/cluster.go index 43f4672e2..f9494593c 100644 --- a/internal/store/cluster.go +++ b/internal/store/cluster.go @@ -44,18 +44,14 @@ func buildRawMetadata(cluster *model.Cluster) (*RawClusterMetadata, error) { return nil, errors.Wrap(err, "unable to marshal ProviderMetadataAWS") } - var provisionerMetadataJSON []byte + provisionerMetadata, err := cluster.GetProvisionerMetadata() + if err != nil { + return nil, errors.Wrap(err, "unable to get ProvisionerMetadata") + } - if cluster.Provisioner == model.ProvisionerKops { - provisionerMetadataJSON, err = json.Marshal(cluster.ProvisionerMetadataKops) - if err != nil { - return nil, errors.Wrap(err, "unable to marshal ProvisionerMetadataKops") - } - } else if cluster.Provisioner == model.ProvisionerEKS { - provisionerMetadataJSON, err = json.Marshal(cluster.ProvisionerMetadataEKS) - if err != nil { - return nil, errors.Wrap(err, "unable to marshal ProvisionerMetadataKops") - } + provisionerMetadataJSON, err := json.Marshal(provisionerMetadata) + if err != nil { + return nil, errors.Wrap(err, "unable to marshal ProvisionerMetadataKops") } utilityMetadataJSON, err := json.Marshal(cluster.UtilityMetadata) @@ -89,6 +85,11 @@ func (r *rawCluster) toCluster() (*model.Cluster, error) { if r.Cluster.ProvisionerMetadataEKS != nil { r.Cluster.Networking = r.Cluster.ProvisionerMetadataEKS.Networking } + } else if r.Provisioner == model.ProvisionerCrossplane { + r.Cluster.ProvisionerMetadataCrossplane, err = model.NewCrossplaneMetadataFromJSON(r.ProvisionerMetadataRaw) + if err != nil { + return nil, err + } } else if r.Provisioner == model.ProvisionerKops { r.Cluster.ProvisionerMetadataKops, err = model.NewKopsMetadata(r.ProvisionerMetadataRaw) if err != nil { From 120576b315d12a6168bb8869f13f301fb9cb21f6 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 4 Apr 2023 15:03:58 +0200 Subject: [PATCH 06/29] dependencies --- go.mod | 9 ++++++--- go.sum | 11 ++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 30609caab..2b3a5109b 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/mattermost/mattermost-cloud go 1.19 +replace github.com/mattermost/mattermost-cloud-crossplane => ../../../.go/src/github.com/mattermost/mattermost-cloud-crossplane + require ( github.com/0xAX/notificator v0.0.0-20220220101646-ee9b8921e557 github.com/Masterminds/squirrel v1.5.3 @@ -33,6 +35,7 @@ require ( github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.7 github.com/mattermost/awat v0.2.0 + github.com/mattermost/mattermost-cloud-crossplane v0.0.0-00010101000000-000000000000 github.com/mattermost/mattermost-operator v1.20.0-rc.2 github.com/mattermost/rotator v0.2.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible @@ -142,11 +145,11 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.30.0 // indirect + k8s.io/klog/v2 v2.40.1 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect - sigs.k8s.io/controller-runtime v0.11.0 // indirect + sigs.k8s.io/controller-runtime v0.11.1 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index fdaebf6d3..ac2a3bac8 100644 --- a/go.sum +++ b/go.sum @@ -2381,8 +2381,9 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.6.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.18.8 h1:8VQxblQqRInpJ+DS2aGgbdWq6xP8UG/jzV6v8cFccOc= k8s.io/kube-aggregator v0.18.8/go.mod h1:CyLoGZB+io8eEwnn+6RbV7QWJQhj8a3TBH8ZM8sLbhI= k8s.io/kube-openapi v0.0.0-20180719232738-d8ea2fe547a4/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= @@ -2420,8 +2421,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PN sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526hjf9iJdbUJ/E= sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ= -sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= +sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU= +sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= @@ -2435,8 +2436,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.0 h1:kDvPBbnPk+qYmkHmSo8vKGp438IASWofnbbUKDE/bv0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 5b64a491d66b6989eaddab869c19abbd22e7211f Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Mon, 10 Apr 2023 10:59:52 +0200 Subject: [PATCH 07/29] provisioner create and check nodes --- cmd/cloud/server.go | 2 ++ internal/provisioner/crossplane_provisioner.go | 17 ++++++++++++----- internal/supervisor/crossplane.go | 13 +++++++------ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/cloud/server.go b/cmd/cloud/server.go index 0991566cb..4a5b17e3a 100644 --- a/cmd/cloud/server.go +++ b/cmd/cloud/server.go @@ -403,6 +403,8 @@ func executeServerCmd(flags serverFlags) error { multiDoer = append(multiDoer, supervisor.NewInstallationDBMigrationSupervisor(sqlStore, awsClient, resourceUtil, instanceID, provisionerObj, eventsProducer, logger)) } + // TODO: crossplane supervisor + // Setup the supervisor to effect any requested changes. It is wrapped in a // scheduler to trigger it periodically in addition to being poked by the API // layer. diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index d840e7c57..b10162639 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -121,14 +121,21 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) // CheckClusterCreated checks if cluster creation finished. func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Cluster) (bool, error) { + object, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Get(context.TODO(), cluster.ID, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrap(err, "error getting crossplane resource information") + } - if false { - // TODO: check status of the cluster - cluster.State = model.ClusterStateStable - return true, provisioner.clusterStore.UpdateCluster(cluster) + ready, err := object.Status.GetReadyCondition() + if err != nil && !errors.Is(err, crossplaneV1Alpha1.ErrConditionNotFound) { + return false, errors.Wrap(err, "error getting crossplane cluster ready status") } - return true, nil + if ready == nil { + return false, nil + } + + return ready.Status == metav1.ConditionTrue, nil } // CreateNodes is no-op. diff --git a/internal/supervisor/crossplane.go b/internal/supervisor/crossplane.go index 82cc71c98..6c38a4f13 100644 --- a/internal/supervisor/crossplane.go +++ b/internal/supervisor/crossplane.go @@ -6,17 +6,16 @@ package supervisor import ( "github.com/mattermost/mattermost-cloud/internal/metrics" + "github.com/mattermost/mattermost-cloud/k8s" "github.com/mattermost/mattermost-cloud/model" log "github.com/sirupsen/logrus" ) -// ClusterSupervisor finds clusters pending work and effects the required changes. -// -// The degree of parallelism is controlled by a weighted semaphore, intended to be shared with -// other clients needing to coordinate background jobs. +// CrossplaneSupervisor finds clusters and syncs their states with the provisioner. type CrossplaneSupervisor struct { store clusterStore - provisioner ClusterProvisionerOption + k8sClient k8s.KubeClient + provisioner ClusterProvisioner eventsProducer eventProducer instanceID string metrics *metrics.CloudMetrics @@ -25,7 +24,8 @@ type CrossplaneSupervisor struct { func NewCrossplaneSupervisor( store clusterStore, - provisioner ClusterProvisionerOption, + k8sClient k8s.KubeClient, + provisioner ClusterProvisioner, eventProducer eventProducer, instanceID string, logger log.FieldLogger, @@ -33,6 +33,7 @@ func NewCrossplaneSupervisor( ) *CrossplaneSupervisor { return &CrossplaneSupervisor{ store: store, + k8sClient: k8sClient, provisioner: provisioner, eventsProducer: eventProducer, instanceID: instanceID, From 4ae290f10d559d22b80ffdab73bdb4f16b8edb43 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Mon, 10 Apr 2023 15:48:00 +0200 Subject: [PATCH 08/29] configurable kube2iam account id --- cmd/cloud/server.go | 2 ++ cmd/cloud/server_flag.go | 6 ++++ .../provisioner/crossplane_provisioner.go | 28 +++++++++++-------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/cmd/cloud/server.go b/cmd/cloud/server.go index 4a5b17e3a..334be0bd0 100644 --- a/cmd/cloud/server.go +++ b/cmd/cloud/server.go @@ -245,6 +245,7 @@ func executeServerCmd(flags serverFlags) error { "enable-crossplane": flags.enableCrossplane, "k8s-use-incluster-config": flags.k8sUseInClusterConfig, "k8s-kubeconfig-path": flags.k8sKubeconfigPath, + "kube2iam-account-id": flags.kube2IAMAccountID, }).Info("Starting Mattermost Provisioning Server") // Warn on settings we consider to be non-production. @@ -309,6 +310,7 @@ func executeServerCmd(flags serverFlags) error { awsClient, provisioningParams, sqlStore, + flags.kube2IAMAccountID, logger, ) } diff --git a/cmd/cloud/server_flag.go b/cmd/cloud/server_flag.go index 2b09ce6a9..6edf3526b 100644 --- a/cmd/cloud/server_flag.go +++ b/cmd/cloud/server_flag.go @@ -181,12 +181,14 @@ type crossplaneOptions struct { enableCrossplane bool k8sUseInClusterConfig bool k8sKubeconfigPath string + kube2IAMAccountID string } func (flags *crossplaneOptions) addFlags(command *cobra.Command) { command.Flags().BoolVar(&flags.enableCrossplane, "enable-crossplane", false, "Whether to use Crossplane to provision resources.") command.Flags().BoolVar(&flags.k8sUseInClusterConfig, "k8s-use-in-cluster-config", false, "Whether to use in-cluster config for k8s client.") command.Flags().StringVar(&flags.k8sKubeconfigPath, "k8s-kubeconfig-path", "", "Path to kubeconfig file for k8s client.") + command.Flags().StringVar(&flags.kube2IAMAccountID, "kube2iam-account-id", "", "AWS account ID for kube2iam.") command.MarkFlagsMutuallyExclusive("k8s-use-in-cluster-config", "k8s-kubeconfig-path") } @@ -194,6 +196,10 @@ func (flags *crossplaneOptions) valid(command *cobra.Command) error { if flags.enableCrossplane && !flags.k8sUseInClusterConfig && flags.k8sKubeconfigPath == "" { return errors.New("k8s-kubeconfig-path or k8s-use-in-cluster-config is required when enable-crossplane is set to true") } + + if flags.enableCrossplane && flags.kube2IAMAccountID == "" { + return errors.New("kube2iam-account-id is required when enable-crossplane is set to true") + } return nil } diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index b10162639..87edf71fd 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -28,11 +28,12 @@ const ( // CrossplaneProvisioner provisions clusters using Crossplane type CrossplaneProvisioner struct { - awsClient aws.AWS - kubeClient *k8s.KubeClient - clusterStore clusterUpdateStore - parameters ProvisioningParams - logger log.FieldLogger + awsClient aws.AWS + kubeClient *k8s.KubeClient + clusterStore clusterUpdateStore + parameters ProvisioningParams + kube2IAMAccountID string + logger log.FieldLogger } var _ supervisor.ClusterProvisioner = (*CrossplaneProvisioner)(nil) @@ -43,14 +44,16 @@ func NewCrossplaneProvisioner( awsClient aws.AWS, parameters ProvisioningParams, clusterStore clusterUpdateStore, + kube2IAMAccountID string, logger log.FieldLogger, ) *CrossplaneProvisioner { return &CrossplaneProvisioner{ - kubeClient: kubeClient, - awsClient: awsClient, - parameters: parameters, - clusterStore: clusterStore, - logger: logger, + kubeClient: kubeClient, + awsClient: awsClient, + parameters: parameters, + clusterStore: clusterStore, + kube2IAMAccountID: kube2IAMAccountID, + logger: logger, } } @@ -93,7 +96,7 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) ID: cluster.ID, Parameters: crossplaneV1Alpha1.EKSSpecParameters{ Version: cluster.ProvisionerMetadataCrossplane.KubernetesVersion, - AccountID: cluster.ProvisionerMetadataCrossplane.AccountID, + AccountID: provisioner.kube2IAMAccountID, Region: cluster.ProvisionerMetadataCrossplane.Region, Environment: "dev", ClusterShortName: cluster.ID, @@ -135,7 +138,8 @@ func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Clu return false, nil } - return ready.Status == metav1.ConditionTrue, nil + // return ready.Status == metav1.ConditionTrue, nil + return true, nil } // CreateNodes is no-op. From 8def3b416dc1996bb255a2a9c34383b7f1076410 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 11 Apr 2023 09:35:44 +0200 Subject: [PATCH 09/29] dev namespace --- internal/provisioner/crossplane_provisioner.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 87edf71fd..29c2362f0 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -23,7 +23,8 @@ const ( CrossplaneProvisionerType = "crossplane" // crossplaneProvisionerNamespace the namespace where the crossplane resources are created. - crossplaneProvisionerNamespace = "mattermost-cloud" + // TODO: change to a proper namespace when tests are done. + crossplaneProvisionerNamespace = "mm-xplane-eks-01" ) // CrossplaneProvisioner provisions clusters using Crossplane @@ -89,7 +90,7 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) ctx := context.TODO() obj := &crossplaneV1Alpha1.MMK8S{ ObjectMeta: metav1.ObjectMeta{ - Name: cluster.ID, + Name: cluster.ProvisionerMetadataCrossplane.Name, Namespace: crossplaneProvisionerNamespace, }, Spec: crossplaneV1Alpha1.EKSSpec{ @@ -98,10 +99,10 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) Version: cluster.ProvisionerMetadataCrossplane.KubernetesVersion, AccountID: provisioner.kube2IAMAccountID, Region: cluster.ProvisionerMetadataCrossplane.Region, - Environment: "dev", + Environment: "dev", // TODO ClusterShortName: cluster.ID, - EndpointPrivateAccess: true, - EndpointPublicAccess: true, + EndpointPrivateAccess: true, // TODO + EndpointPublicAccess: false, // TODO VpcID: cluster.ProvisionerMetadataCrossplane.VPC, SubnetIds: cluster.ProvisionerMetadataCrossplane.PublicSubnets, PrivateSubnetIds: cluster.ProvisionerMetadataCrossplane.PrivateSubnets, @@ -178,7 +179,8 @@ func (provisioner *CrossplaneProvisioner) DeleteCluster(cluster *model.Cluster) err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ID, metav1.DeleteOptions{}) if err != nil { - return false, errors.Wrap(err, "failed to delete crossplane resource") + provisioner.logger.WithError(err).Error("Failed to delete crossplane resource") + // return false, errors.Wrap(err, "failed to delete crossplane resource") } err = provisioner.awsClient.ReleaseVpc(cluster, logger) From 21a7286321af04f11057d3da62f7144a05f13bee Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Wed, 12 Apr 2023 11:37:58 +0200 Subject: [PATCH 10/29] fix: deletion typo --- internal/provisioner/crossplane_provisioner.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 29c2362f0..f9380010b 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -15,6 +15,7 @@ import ( "github.com/mattermost/mattermost-cloud/model" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -126,7 +127,7 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) // CheckClusterCreated checks if cluster creation finished. func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Cluster) (bool, error) { object, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Get(context.TODO(), cluster.ID, metav1.GetOptions{}) - if err != nil { + if err != nil && !k8sErrors.IsNotFound(err) { return false, errors.Wrap(err, "error getting crossplane resource information") } @@ -177,7 +178,7 @@ func (provisioner *CrossplaneProvisioner) ResizeCluster(cluster *model.Cluster) func (provisioner *CrossplaneProvisioner) DeleteCluster(cluster *model.Cluster) (bool, error) { logger := provisioner.logger.WithField("cluster", cluster.ID) - err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ID, metav1.DeleteOptions{}) + err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ProvisionerMetadataCrossplane.Name, metav1.DeleteOptions{}) if err != nil { provisioner.logger.WithError(err).Error("Failed to delete crossplane resource") // return false, errors.Wrap(err, "failed to delete crossplane resource") From 56c5bd653ade28aafd1e442538a9e26b73aee81b Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Thu, 13 Apr 2023 08:59:30 +0200 Subject: [PATCH 11/29] remove test files --- xrd/main.go | 81 ---------------------------------------------------- xrd/xrd.yaml | 33 --------------------- 2 files changed, 114 deletions(-) delete mode 100644 xrd/main.go delete mode 100644 xrd/xrd.yaml diff --git a/xrd/main.go b/xrd/main.go deleted file mode 100644 index 298b8605c..000000000 --- a/xrd/main.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. -// - -package main - -import ( - "encoding/json" - - "github.com/mattermost/mattermost-cloud/internal/provisioner/crossplane" - "github.com/mattermost/mattermost-cloud/k8s" - "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func main() { - // ctx := context.TODO() - logger := logrus.New() - namespace := "testing" - obj := crossplane.EKS{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: namespace, - // Labels: map[string]string{"test": "test"}, - // Annotations: , - }, - Spec: crossplane.EKSSpec{}, - } - - // obj := corev1.Pod{ - // // TypeMeta: metav1.TypeMeta{ - // // Kind: "Pod", - // // APIVersion: "v1", - // // }, - // ObjectMeta: metav1.ObjectMeta{ - // Name: "test-nginx", - // Namespace: namespace, - // }, - // Spec: corev1.PodSpec{ - // Containers: []corev1.Container{{ - // Image: "nginx", - // }}, - // }, - // } - - // cfg, err := rest.InClusterConfig() - // if err != nil { - // panic(err) - // } - - // client, err := k8s.NewFromConfig(cfg, logger) - // if err != nil { - // panic(err) - // } - - objBytes, err := json.Marshal(obj) - if err != nil { - panic(err) - } - - client, err := k8s.NewFromFile("/Users/fmartingr/.kube/config_kind_provisioner", logger) - if err != nil { - panic(err) - } - - req := client.Clientset.CoreV1().RESTClient(). - Post(). - Resource("cloud.mattermost.sio/mmK8ss"). - Namespace(obj.Namespace). - Name(obj.Name). - Body(objBytes) - - output, execErr := client.RemoteCommand("POST", req.URL()) - - if execErr != nil { - logger.WithError(execErr).Error("failed") - } else { - logger.Debugf(string(output)) - } - -} diff --git a/xrd/xrd.yaml b/xrd/xrd.yaml deleted file mode 100644 index 684b4dc34..000000000 --- a/xrd/xrd.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: cloud.mattermost.io/v1alpha1 -kind: mmK8s -metadata: - name: b07c8248b874b3821af37d81d3 -spec: - compositionSelector: - matchLabels: - provider: aws - service: eks - id: b07c8248b874b3821af37d81d3 - parameters: - version: "1.23" - account_id: "926412419614" - region: us-east-1 - environment: dev - cluster_short_name: crossplane-test - endpoint_private_access: true - endpoint_public_access: false - vpc_id: vpc-09d44077df9934f96 - subnet_ids: - - "subnet-0866bf0625a68488a" - - "subnet-0f74e2dfcdcdc18b2" - - "subnet-0e123b8281481294a" - - "subnet-0ccc5bf96d05d2a3a" - private_subnet_ids: - - "subnet-0e123b8281481294a" - - "subnet-0ccc5bf96d05d2a3a" - node_count: 1 - instance_type: t3.large - image_id: ami-0bc89720a2e35652e - launch_template_version: "2" - resourceConfig: - providerConfigName: crossplane-provider-config From ab4bd037c985b073b612b7d267ceae76deaea71b Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Thu, 13 Apr 2023 09:00:52 +0200 Subject: [PATCH 12/29] fix: missing spec attributes --- internal/provisioner/crossplane_provisioner.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index f9380010b..40b9aad62 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -95,6 +95,12 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) Namespace: crossplaneProvisionerNamespace, }, Spec: crossplaneV1Alpha1.EKSSpec{ + CompositionSelector: crossplaneV1Alpha1.EKSSpecCompositionSelector{ + MatchLabels: crossplaneV1Alpha1.EKSSpecCompositionSelectorMatchLabels{ + Provider: "aws", + Service: "eks", + }, + }, ID: cluster.ID, Parameters: crossplaneV1Alpha1.EKSSpecParameters{ Version: cluster.ProvisionerMetadataCrossplane.KubernetesVersion, @@ -112,7 +118,9 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) ImageID: cluster.ProvisionerMetadataCrossplane.AMI, LaunchTemplateVersion: *cluster.ProvisionerMetadataCrossplane.LaunchTemplateVersion, }, - ResourceConfig: crossplaneV1Alpha1.EKSSpecResourceConfig{}, + ResourceConfig: crossplaneV1Alpha1.EKSSpecResourceConfig{ + ProviderConfigName: "crossplane-provider-config", + }, }, } @@ -126,12 +134,12 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) // CheckClusterCreated checks if cluster creation finished. func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Cluster) (bool, error) { - object, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Get(context.TODO(), cluster.ID, metav1.GetOptions{}) + resource, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Get(context.TODO(), cluster.ID, metav1.GetOptions{}) if err != nil && !k8sErrors.IsNotFound(err) { return false, errors.Wrap(err, "error getting crossplane resource information") } - ready, err := object.Status.GetReadyCondition() + ready, err := resource.Status.GetReadyCondition() if err != nil && !errors.Is(err, crossplaneV1Alpha1.ErrConditionNotFound) { return false, errors.Wrap(err, "error getting crossplane cluster ready status") } @@ -140,8 +148,7 @@ func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Clu return false, nil } - // return ready.Status == metav1.ConditionTrue, nil - return true, nil + return ready.Status == metav1.ConditionTrue, nil } // CreateNodes is no-op. From 727e083a2bec321a73635df8707d0469bee3dcf3 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Fri, 14 Apr 2023 09:38:59 +0200 Subject: [PATCH 13/29] fix: store kubeadm account id in metadata --- internal/provisioner/crossplane_provisioner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 40b9aad62..4832eadd6 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -82,6 +82,7 @@ func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) cluster.ProvisionerMetadataCrossplane.VPC = resources.VpcID cluster.ProvisionerMetadataCrossplane.PublicSubnets = resources.PublicSubnetsIDs cluster.ProvisionerMetadataCrossplane.PrivateSubnets = resources.PrivateSubnetIDs + cluster.ProvisionerMetadataCrossplane.AccountID = provisioner.kube2IAMAccountID return true } From 39478502008efcf60964d251218341ac718d642a Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 08:48:34 +0200 Subject: [PATCH 14/29] default node count --- model/crossplane_metadata.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/model/crossplane_metadata.go b/model/crossplane_metadata.go index 96cce8c83..8d340a8d7 100644 --- a/model/crossplane_metadata.go +++ b/model/crossplane_metadata.go @@ -17,10 +17,13 @@ const ( defaultLaunchTemplateVersion = "2" // defaultKubernetesVersion is the default Kubernetes version to use. - defaultKubernetesVersion = "latest" + defaultKubernetesVersion = "1.23" // defaultInstanceType is the default AWS instance type to use. defaultInstanceType = "t3.medium" + + // defaultNodeCount is the default number of nodes to use. + defaultNodeCount = 1 ) type CrossplaneMetadata struct { @@ -76,6 +79,10 @@ func (m *CrossplaneMetadata) SetDefaults() { if m.InstanceType == "" { m.InstanceType = defaultInstanceType } + + if m.NodeCount == 0 { + m.NodeCount = defaultNodeCount + } } // CrossplaneMetadataRequestedState is the requested state for crossplane metadata. From d9cab627d10866577c3e9cad100d46ec453f588d Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 08:48:51 +0200 Subject: [PATCH 15/29] ami setup --- .../provisioner/crossplane_provisioner.go | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 4832eadd6..4c7dc9f5f 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -60,29 +60,47 @@ func NewCrossplaneProvisioner( } // PrepareCluster prepares the cluster for provisioning by assigning it a name (if not manually -// provided) and claiming the VPC required for the cluster to be provisioned. +// provided) and claiming the VPC required for the cluster to be provisioned and checking and +// setting any creation request parameters. func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) bool { - if cluster.ProvisionerMetadataCrossplane.Name == "" { - cluster.ProvisionerMetadataCrossplane.Name = fmt.Sprintf("%s-crossplane-k8s-local", cluster.ID) + logger := provisioner.logger.WithField("cluster", cluster.ID) + metadata := cluster.ProvisionerMetadataCrossplane + + if metadata.ChangeRequest.AMI != "" && metadata.ChangeRequest.AMI != "latest" { + var isAMIValid bool + isAMIValid, err := provisioner.awsClient.IsValidAMI(metadata.ChangeRequest.AMI, logger) + if err != nil { + logger.WithError(err).Errorf("error checking the AWS AMI image %s", metadata.ChangeRequest.AMI) + return false + } + if !isAMIValid { + logger.Errorf("invalid AWS AMI image %s", metadata.ChangeRequest.AMI) + return false + } + metadata.AMI = metadata.ChangeRequest.AMI + } + + if metadata.Name == "" { + metadata.Name = fmt.Sprintf("%s-crossplane-k8s-local", cluster.ID) } var ( resources aws.ClusterResources err error ) - if cluster.ProvisionerMetadataCrossplane.VPC == "" { + if metadata.VPC == "" { resources, err = provisioner.awsClient.GetAndClaimVpcResources(cluster, provisioner.parameters.Owner, provisioner.logger) } else { - resources, err = provisioner.awsClient.ClaimVPC(cluster.ProvisionerMetadataCrossplane.VPC, cluster, provisioner.parameters.Owner, provisioner.logger) + resources, err = provisioner.awsClient.ClaimVPC(metadata.VPC, cluster, provisioner.parameters.Owner, provisioner.logger) } if err != nil { - provisioner.logger.WithError(err).WithField("vpc", cluster.ProvisionerMetadataCrossplane.VPC).Error("Failed to claim VPC resources") + provisioner.logger.WithError(err).WithField("vpc", metadata.VPC).Error("Failed to claim VPC resources") return false } - cluster.ProvisionerMetadataCrossplane.VPC = resources.VpcID - cluster.ProvisionerMetadataCrossplane.PublicSubnets = resources.PublicSubnetsIDs - cluster.ProvisionerMetadataCrossplane.PrivateSubnets = resources.PrivateSubnetIDs - cluster.ProvisionerMetadataCrossplane.AccountID = provisioner.kube2IAMAccountID + metadata.VPC = resources.VpcID + metadata.PublicSubnets = resources.PublicSubnetsIDs + metadata.PrivateSubnets = resources.PrivateSubnetIDs + metadata.AccountID = provisioner.kube2IAMAccountID return true } @@ -90,6 +108,8 @@ func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) // CreateCluster creates the Crossplane cluster resource in the kubernetes CNC cluster. func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) error { ctx := context.TODO() + // logger := provisioner.logger.WithField("cluster", cluster.ID) + obj := &crossplaneV1Alpha1.MMK8S{ ObjectMeta: metav1.ObjectMeta{ Name: cluster.ProvisionerMetadataCrossplane.Name, From bf2c57f1d7b047bba5f72980219e16521822e9f0 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 17:43:13 +0200 Subject: [PATCH 16/29] fix: cluster version default to empty --- cmd/cloud/cluster_flag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cloud/cluster_flag.go b/cmd/cloud/cluster_flag.go index d0b657cac..2a41ee9ac 100644 --- a/cmd/cloud/cluster_flag.go +++ b/cmd/cloud/cluster_flag.go @@ -41,7 +41,7 @@ type createRequestOptions struct { func (flags *createRequestOptions) addFlags(command *cobra.Command) { command.Flags().StringVar(&flags.provider, "provider", "aws", "Cloud provider hosting the cluster.") - command.Flags().StringVar(&flags.version, "version", "latest", "The Kubernetes version to target. Use 'latest' or versions such as '1.16.10'.") + command.Flags().StringVar(&flags.version, "version", "", "The Kubernetes version to target. Use 'latest' or versions such as '1.16.10'.") command.Flags().StringVar(&flags.ami, "ami", "", "The AMI to use for the cluster hosts.") command.Flags().StringVar(&flags.zones, "zones", "us-east-1a", "The zones where the cluster will be deployed. Use commas to separate multiple zones.") command.Flags().BoolVar(&flags.allowInstallations, "allow-installations", true, "Whether the cluster will allow for new installations to be scheduled.") From 9eba690ae21409183444ad6a35a72d087d320b28 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 17:43:28 +0200 Subject: [PATCH 17/29] fix: set defaults after applying create request --- internal/api/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/cluster.go b/internal/api/cluster.go index 6dc911f3d..c85587a46 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -125,8 +125,8 @@ func handleCreateCluster(c *Context, w http.ResponseWriter, r *http.Request) { // TODO: Defaults to first zone in the AWS metadata for now. Region: cluster.ProviderMetadataAWS.Zones[0], } - cluster.ProvisionerMetadataCrossplane.SetDefaults() cluster.ProvisionerMetadataCrossplane.ApplyClusterCreateRequest(createClusterRequest) + cluster.ProvisionerMetadataCrossplane.SetDefaults() } else { cluster.ProvisionerMetadataKops = &model.KopsMetadata{} cluster.ProvisionerMetadataKops.ApplyClusterCreateRequest(createClusterRequest) From 46a25c5e9ce863f5443a956da69cc25962567eff Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 17:43:47 +0200 Subject: [PATCH 18/29] apply all missing create request parameters --- internal/provisioner/crossplane_provisioner.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 4c7dc9f5f..db5b7139d 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -66,6 +66,7 @@ func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) logger := provisioner.logger.WithField("cluster", cluster.ID) metadata := cluster.ProvisionerMetadataCrossplane + // Validate the cluster request AMI if provided. if metadata.ChangeRequest.AMI != "" && metadata.ChangeRequest.AMI != "latest" { var isAMIValid bool isAMIValid, err := provisioner.awsClient.IsValidAMI(metadata.ChangeRequest.AMI, logger) @@ -77,13 +78,17 @@ func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) logger.Errorf("invalid AWS AMI image %s", metadata.ChangeRequest.AMI) return false } - metadata.AMI = metadata.ChangeRequest.AMI } if metadata.Name == "" { metadata.Name = fmt.Sprintf("%s-crossplane-k8s-local", cluster.ID) } + if err := metadata.ApplyChangeRequest(); err != nil { + logger.WithError(err).Error("Failed to apply change request") + return false + } + var ( resources aws.ClusterResources err error @@ -145,6 +150,9 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) }, } + // bites, err := json.Marshal(obj) + // provisioner.logger.Warnf("obj: %s", string(bites)) + _, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Create(ctx, obj, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "error creating object") From a56a0ccf2c34c7dbe5d28f4fc4dfc4d493fba54e Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 17:44:10 +0200 Subject: [PATCH 19/29] handle crossplane in creation request parameters --- model/cluster_request.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/cluster_request.go b/model/cluster_request.go index f84d50ad7..2c7a099f3 100644 --- a/model/cluster_request.go +++ b/model/cluster_request.go @@ -73,7 +73,7 @@ func (request *CreateClusterRequest) SetDefaults() { request.Provisioner = ProvisionerKops } if len(request.Version) == 0 { - if request.Provisioner == ProvisionerEKS { + if request.Provisioner == ProvisionerEKS || request.Provisioner == ProvisionerCrossplane { request.Version = "1.23" } else { request.Version = "latest" @@ -81,7 +81,7 @@ func (request *CreateClusterRequest) SetDefaults() { } if len(request.Zones) == 0 { - if request.Provisioner == ProvisionerEKS { + if request.Provisioner == ProvisionerEKS || request.Provisioner == ProvisionerCrossplane { request.Zones = []string{"us-east-1a", "us-east-1b"} } else { request.Zones = []string{"us-east-1a"} From 693ebcd64384d8493c6a8731c230feb14c403a68 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Tue, 18 Apr 2023 17:44:20 +0200 Subject: [PATCH 20/29] crossplane defaults --- model/crossplane_metadata.go | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/model/crossplane_metadata.go b/model/crossplane_metadata.go index 8d340a8d7..84258599f 100644 --- a/model/crossplane_metadata.go +++ b/model/crossplane_metadata.go @@ -20,7 +20,7 @@ const ( defaultKubernetesVersion = "1.23" // defaultInstanceType is the default AWS instance type to use. - defaultInstanceType = "t3.medium" + defaultInstanceType = "t3.large" // defaultNodeCount is the default number of nodes to use. defaultNodeCount = 1 @@ -44,9 +44,7 @@ type CrossplaneMetadata struct { func (m *CrossplaneMetadata) ApplyClusterCreateRequest(createRequest *CreateClusterRequest) bool { m.ChangeRequest = &CrossplaneMetadataRequestedState{ AMI: createRequest.AMI, - ClusterRoleARN: createRequest.ClusterRoleARN, MaxPodsPerNode: createRequest.MaxPodsPerNode, - Networking: createRequest.Networking, Version: createRequest.Version, VPC: createRequest.VPC, } @@ -72,6 +70,7 @@ func (m *CrossplaneMetadata) SetDefaults() { m.LaunchTemplateVersion = ptr.String(defaultLaunchTemplateVersion) } + // Safeguard, should be set by the cluster creation request. if m.KubernetesVersion == "" { m.KubernetesVersion = defaultKubernetesVersion } @@ -85,12 +84,38 @@ func (m *CrossplaneMetadata) SetDefaults() { } } +// ApplyChangeRequest applies the change request to the metadata if the values are provided. +func (m *CrossplaneMetadata) ApplyChangeRequest() error { + if m.ChangeRequest == nil { + return nil + } + + if m.ChangeRequest.AMI != "" { + m.AMI = m.ChangeRequest.AMI + } + + if m.ChangeRequest.Version != "" { + m.KubernetesVersion = m.ChangeRequest.Version + } + + if m.ChangeRequest.NodeCount != 0 { + m.NodeCount = m.ChangeRequest.NodeCount + } + + if m.ChangeRequest.VPC != "" { + m.VPC = m.ChangeRequest.VPC + } + + return nil +} + // CrossplaneMetadataRequestedState is the requested state for crossplane metadata. type CrossplaneMetadataRequestedState struct { AMI string ClusterRoleARN string LaunchTemplateVersion *string MaxPodsPerNode int64 + NodeCount int64 Networking string Version string VPC string From 5834475113118d7609a4c99b1961eb3fe867a93f Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Fri, 21 Apr 2023 08:02:14 +0200 Subject: [PATCH 21/29] use region instead of zone --- internal/api/cluster.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/api/cluster.go b/internal/api/cluster.go index c85587a46..eeacd7f51 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -123,7 +123,9 @@ func handleCreateCluster(c *Context, w http.ResponseWriter, r *http.Request) { } else if createClusterRequest.Provisioner == model.ProvisionerCrossplane { cluster.ProvisionerMetadataCrossplane = &model.CrossplaneMetadata{ // TODO: Defaults to first zone in the AWS metadata for now. - Region: cluster.ProviderMetadataAWS.Zones[0], + // FIXME: It gets the region from the first zone, but it should be inherited from the + // VPC or it should use zones instead of regions. + Region: cluster.ProviderMetadataAWS.Zones[0][:len(cluster.ProviderMetadataAWS.Zones[0])-1], } cluster.ProvisionerMetadataCrossplane.ApplyClusterCreateRequest(createClusterRequest) cluster.ProvisionerMetadataCrossplane.SetDefaults() From cd5a1757401290bc79ca14a1b3062fbee4650b14 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Fri, 21 Apr 2023 11:45:19 +0200 Subject: [PATCH 22/29] filter out subnets from other zones from the resource --- .../provisioner/crossplane_provisioner.go | 33 ++++++++++++++++--- internal/tools/aws/cluster_management.go | 4 +++ internal/tools/utils/generics.go | 15 +++++++++ model/crossplane_metadata.go | 2 +- 4 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 internal/tools/utils/generics.go diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index db5b7139d..1c525efed 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -11,6 +11,7 @@ import ( crossplaneV1Alpha1 "github.com/mattermost/mattermost-cloud-crossplane/apis/crossplane/v1alpha1" "github.com/mattermost/mattermost-cloud/internal/supervisor" "github.com/mattermost/mattermost-cloud/internal/tools/aws" + "github.com/mattermost/mattermost-cloud/internal/tools/utils" "github.com/mattermost/mattermost-cloud/k8s" "github.com/mattermost/mattermost-cloud/model" "github.com/pkg/errors" @@ -102,9 +103,19 @@ func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) provisioner.logger.WithError(err).WithField("vpc", metadata.VPC).Error("Failed to claim VPC resources") return false } + metadata.VPC = resources.VpcID - metadata.PublicSubnets = resources.PublicSubnetsIDs - metadata.PrivateSubnets = resources.PrivateSubnetIDs + for _, subnet := range resources.PublicSubnets { + if utils.Contains[string](cluster.ProviderMetadataAWS.Zones, *subnet.AvailabilityZone) { + metadata.Subnets = append(metadata.Subnets, *subnet.SubnetId) + } + } + for _, subnet := range resources.PrivateSubnets { + if utils.Contains[string](cluster.ProviderMetadataAWS.Zones, *subnet.AvailabilityZone) { + metadata.Subnets = append(metadata.Subnets, *subnet.SubnetId) + metadata.PrivateSubnets = append(metadata.PrivateSubnets, *subnet.SubnetId) + } + } metadata.AccountID = provisioner.kube2IAMAccountID return true @@ -137,7 +148,7 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) EndpointPrivateAccess: true, // TODO EndpointPublicAccess: false, // TODO VpcID: cluster.ProvisionerMetadataCrossplane.VPC, - SubnetIds: cluster.ProvisionerMetadataCrossplane.PublicSubnets, + SubnetIds: cluster.ProvisionerMetadataCrossplane.Subnets, PrivateSubnetIds: cluster.ProvisionerMetadataCrossplane.PrivateSubnets, NodeCount: int(cluster.ProvisionerMetadataCrossplane.NodeCount), InstanceType: cluster.ProvisionerMetadataCrossplane.InstanceType, @@ -163,16 +174,30 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) // CheckClusterCreated checks if cluster creation finished. func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Cluster) (bool, error) { - resource, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Get(context.TODO(), cluster.ID, metav1.GetOptions{}) + resources, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).List(context.TODO(), metav1.ListOptions{ + FieldSelector: fmt.Sprintf("metadata.name=%s", cluster.ProvisionerMetadataCrossplane.Name), + }) //.Get(context.TODO(), cluster.ID, metav1.GetOptions{}) if err != nil && !k8sErrors.IsNotFound(err) { return false, errors.Wrap(err, "error getting crossplane resource information") } + if len(resources.Items) == 0 { + return false, fmt.Errorf("no crossplane resource found") + } + + if len(resources.Items) > 1 { + return false, fmt.Errorf("expected one eks cluster, found %d", len(resources.Items)) + } + + resource := resources.Items[0] ready, err := resource.Status.GetReadyCondition() if err != nil && !errors.Is(err, crossplaneV1Alpha1.ErrConditionNotFound) { return false, errors.Wrap(err, "error getting crossplane cluster ready status") } + provisioner.logger.Warnf("Conditions: %v", resource.Status.Conditions) + provisioner.logger.Warnf("Ready: %v", ready) + if ready == nil { return false, nil } diff --git a/internal/tools/aws/cluster_management.go b/internal/tools/aws/cluster_management.go index 21ced2e1f..aa3e1312f 100644 --- a/internal/tools/aws/cluster_management.go +++ b/internal/tools/aws/cluster_management.go @@ -21,7 +21,9 @@ import ( type ClusterResources struct { VpcID string VpcCIDR string + PrivateSubnets []ec2Types.Subnet PrivateSubnetIDs []string + PublicSubnets []ec2Types.Subnet PublicSubnetsIDs []string MasterSecurityGroupIDs []string WorkerSecurityGroupIDs []string @@ -138,6 +140,7 @@ func (a *Client) getClusterResourcesForVPC(vpcID, vpcCIDR string, logger log.Fie } for _, subnet := range privateSubnets { + clusterResources.PrivateSubnets = append(clusterResources.PrivateSubnets, subnet) clusterResources.PrivateSubnetIDs = append(clusterResources.PrivateSubnetIDs, *subnet.SubnetId) } @@ -152,6 +155,7 @@ func (a *Client) getClusterResourcesForVPC(vpcID, vpcCIDR string, logger log.Fie } for _, subnet := range publicSubnets { + clusterResources.PublicSubnets = append(clusterResources.PublicSubnets, subnet) clusterResources.PublicSubnetsIDs = append(clusterResources.PublicSubnetsIDs, *subnet.SubnetId) } diff --git a/internal/tools/utils/generics.go b/internal/tools/utils/generics.go new file mode 100644 index 000000000..084c17b99 --- /dev/null +++ b/internal/tools/utils/generics.go @@ -0,0 +1,15 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +// + +package utils + +// Contains generic function to check if an item is in an array of items +func Contains[T comparable](haystack []T, needle T) bool { + for _, item := range haystack { + if item == needle { + return true + } + } + return false +} diff --git a/model/crossplane_metadata.go b/model/crossplane_metadata.go index 84258599f..9f3567381 100644 --- a/model/crossplane_metadata.go +++ b/model/crossplane_metadata.go @@ -34,7 +34,7 @@ type CrossplaneMetadata struct { KubernetesVersion string LaunchTemplateVersion *string PrivateSubnets []string - PublicSubnets []string + Subnets []string Region string VPC string InstanceType string From 41e6e30a8d477d420a8106a0ada5b7a4527a0c9b Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Fri, 21 Apr 2023 11:49:08 +0200 Subject: [PATCH 23/29] proper deletion errors --- internal/provisioner/crossplane_provisioner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 1c525efed..78948d28b 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -241,8 +241,7 @@ func (provisioner *CrossplaneProvisioner) DeleteCluster(cluster *model.Cluster) err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ProvisionerMetadataCrossplane.Name, metav1.DeleteOptions{}) if err != nil { - provisioner.logger.WithError(err).Error("Failed to delete crossplane resource") - // return false, errors.Wrap(err, "failed to delete crossplane resource") + return false, errors.Wrap(err, "failed to delete crossplane resource") } err = provisioner.awsClient.ReleaseVpc(cluster, logger) From 800b1262554945932e42aef3597dd7e2e08207f4 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Fri, 21 Apr 2023 12:22:05 +0200 Subject: [PATCH 24/29] remove debug log lines --- internal/provisioner/crossplane_provisioner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 78948d28b..76d2fdad3 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -195,8 +195,7 @@ func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Clu return false, errors.Wrap(err, "error getting crossplane cluster ready status") } - provisioner.logger.Warnf("Conditions: %v", resource.Status.Conditions) - provisioner.logger.Warnf("Ready: %v", ready) + // TODO: Check if cluster has been pending creation for too long if ready == nil { return false, nil From 41e907a1471eae512e059a7858d59a82a2b4d1f1 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Sun, 23 Apr 2023 18:57:06 +0200 Subject: [PATCH 25/29] inversed delete operations --- internal/provisioner/crossplane_provisioner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 76d2fdad3..e479faf4f 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -238,14 +238,14 @@ func (provisioner *CrossplaneProvisioner) ResizeCluster(cluster *model.Cluster) func (provisioner *CrossplaneProvisioner) DeleteCluster(cluster *model.Cluster) (bool, error) { logger := provisioner.logger.WithField("cluster", cluster.ID) - err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ProvisionerMetadataCrossplane.Name, metav1.DeleteOptions{}) + err := provisioner.awsClient.ReleaseVpc(cluster, logger) if err != nil { - return false, errors.Wrap(err, "failed to delete crossplane resource") + return false, errors.Wrap(err, "failed to release cluster VPC") } - err = provisioner.awsClient.ReleaseVpc(cluster, logger) + err = provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).Delete(context.TODO(), cluster.ProvisionerMetadataCrossplane.Name, metav1.DeleteOptions{}) if err != nil { - return false, errors.Wrap(err, "failed to release cluster VPC") + return false, errors.Wrap(err, "failed to delete crossplane resource") } logger.Info("Successfully deleted Crossplane cluster") From 8d76839f4a38d3bc652d628771a04d4bcba9f26c Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Sun, 23 Apr 2023 18:57:38 +0200 Subject: [PATCH 26/29] force cluster name --- internal/provisioner/crossplane_provisioner.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index e479faf4f..31ef216fb 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -81,9 +81,7 @@ func (provisioner *CrossplaneProvisioner) PrepareCluster(cluster *model.Cluster) } } - if metadata.Name == "" { - metadata.Name = fmt.Sprintf("%s-crossplane-k8s-local", cluster.ID) - } + metadata.Name = fmt.Sprintf("%s-crossplane-k8s-local", cluster.ID) if err := metadata.ApplyChangeRequest(); err != nil { logger.WithError(err).Error("Failed to apply change request") From 377bab26a6309f1eb7a81dde9ad084f4d1cf2a2c Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Sun, 23 Apr 2023 18:57:47 +0200 Subject: [PATCH 27/29] removed old comment --- internal/provisioner/crossplane_provisioner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 31ef216fb..e916788af 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -174,7 +174,7 @@ func (provisioner *CrossplaneProvisioner) CreateCluster(cluster *model.Cluster) func (provisioner *CrossplaneProvisioner) CheckClusterCreated(cluster *model.Cluster) (bool, error) { resources, err := provisioner.kubeClient.CrossplaneClient.CloudV1alpha1().MMK8Ss(crossplaneProvisionerNamespace).List(context.TODO(), metav1.ListOptions{ FieldSelector: fmt.Sprintf("metadata.name=%s", cluster.ProvisionerMetadataCrossplane.Name), - }) //.Get(context.TODO(), cluster.ID, metav1.GetOptions{}) + }) if err != nil && !k8sErrors.IsNotFound(err) { return false, errors.Wrap(err, "error getting crossplane resource information") } From 109e58c599c4c1cd1d0425a77ccb4bea3b78469c Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Sun, 23 Apr 2023 19:01:29 +0200 Subject: [PATCH 28/29] reused k8s connection from eks provisioner --- .../provisioner/crossplane_provisioner.go | 70 +++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index e916788af..32da5aadb 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -7,6 +7,7 @@ package provisioner import ( "context" "fmt" + "os" crossplaneV1Alpha1 "github.com/mattermost/mattermost-cloud-crossplane/apis/crossplane/v1alpha1" "github.com/mattermost/mattermost-cloud/internal/supervisor" @@ -18,6 +19,7 @@ import ( log "github.com/sirupsen/logrus" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" ) const ( @@ -33,7 +35,7 @@ const ( type CrossplaneProvisioner struct { awsClient aws.AWS kubeClient *k8s.KubeClient - clusterStore clusterUpdateStore + databaseStore model.ClusterUtilityDatabaseStoreInterface parameters ProvisioningParams kube2IAMAccountID string logger log.FieldLogger @@ -46,7 +48,7 @@ func NewCrossplaneProvisioner( kubeClient *k8s.KubeClient, awsClient aws.AWS, parameters ProvisioningParams, - clusterStore clusterUpdateStore, + databaseStore model.ClusterUtilityDatabaseStoreInterface, kube2IAMAccountID string, logger log.FieldLogger, ) *CrossplaneProvisioner { @@ -54,7 +56,7 @@ func NewCrossplaneProvisioner( kubeClient: kubeClient, awsClient: awsClient, parameters: parameters, - clusterStore: clusterStore, + databaseStore: databaseStore, kube2IAMAccountID: kube2IAMAccountID, logger: logger, } @@ -214,7 +216,19 @@ func (provisioner *CrossplaneProvisioner) CheckNodesCreated(cluster *model.Clust // ProvisionCluster func (provisioner *CrossplaneProvisioner) ProvisionCluster(cluster *model.Cluster) error { - return nil + logger := provisioner.logger.WithField("cluster", cluster.ID) + + metadata := cluster.ProvisionerMetadataCrossplane + if metadata == nil { + return errors.New("expected metadata to be present") + } + + kubeConfigPath, err := provisioner.getKubeConfigPath(cluster) + if err != nil { + return errors.Wrap(err, "failed to get kubeconfig file path") + } + + return provisionCluster(cluster, kubeConfigPath, provisioner.awsClient, provisioner.parameters, provisioner.databaseStore, logger) } // UpgradeCluster is no-op. @@ -255,3 +269,51 @@ func (provisioner *CrossplaneProvisioner) DeleteCluster(cluster *model.Cluster) func (provisioner *CrossplaneProvisioner) RefreshClusterMetadata(cluster *model.Cluster) error { return nil } + +func (provisioner *CrossplaneProvisioner) getKubeConfigPath(cluster *model.Cluster) (string, error) { + clusterName := "cluster-" + cluster.ID + eksCluster, err := provisioner.awsClient.GetActiveEKSCluster(clusterName) + if err != nil { + return "", errors.Wrap(err, "failed to get EKS cluster") + } + if eksCluster == nil { + return "", errors.New("EKS cluster not ready") + } + + kubeconfig, err := newEKSKubeConfig(eksCluster, provisioner.awsClient) + if err != nil { + return "", errors.Wrap(err, "failed to create kubeconfig") + } + + kubeconfigFile, err := os.CreateTemp("", clusterName) + if err != nil { + return "", errors.Wrap(err, "failed to create kubeconfig tempfile") + } + defer kubeconfigFile.Close() + + rawKubeconfig, err := clientcmd.Write(kubeconfig) + if err != nil { + return "", errors.Wrap(err, "failed to serialize kubeconfig") + } + _, err = kubeconfigFile.Write(rawKubeconfig) + if err != nil { + return "", errors.Wrap(err, "failed to write kubeconfig") + } + + return kubeconfigFile.Name(), nil +} + +func (provisioner *CrossplaneProvisioner) getKubeClient(cluster *model.Cluster) (*k8s.KubeClient, error) { + configLocation, err := provisioner.getKubeConfigPath(cluster) + if err != nil { + return nil, errors.Wrap(err, "failed to get kube config") + } + + var k8sClient *k8s.KubeClient + k8sClient, err = k8s.NewFromFile(configLocation, provisioner.logger) + if err != nil { + return nil, errors.Wrap(err, "failed to create k8s client from file") + } + + return k8sClient, nil +} From 09bc46a252b8da7ac483d29ef4719199c88e23ef Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Sun, 23 Apr 2023 20:40:26 +0200 Subject: [PATCH 29/29] cluster name const --- internal/provisioner/crossplane_provisioner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/provisioner/crossplane_provisioner.go b/internal/provisioner/crossplane_provisioner.go index 32da5aadb..f78b8501d 100644 --- a/internal/provisioner/crossplane_provisioner.go +++ b/internal/provisioner/crossplane_provisioner.go @@ -29,6 +29,10 @@ const ( // crossplaneProvisionerNamespace the namespace where the crossplane resources are created. // TODO: change to a proper namespace when tests are done. crossplaneProvisionerNamespace = "mm-xplane-eks-01" + + // crossplaneEKSClusterNameFormat is the format for the name of the crossplane EKS cluster. + // The variable is the cluster ID. + crossplaneEKSClusterNameFormat = "cluster-%s" ) // CrossplaneProvisioner provisions clusters using Crossplane @@ -271,7 +275,7 @@ func (provisioner *CrossplaneProvisioner) RefreshClusterMetadata(cluster *model. } func (provisioner *CrossplaneProvisioner) getKubeConfigPath(cluster *model.Cluster) (string, error) { - clusterName := "cluster-" + cluster.ID + clusterName := fmt.Sprintf(crossplaneEKSClusterNameFormat, cluster.ID) eksCluster, err := provisioner.awsClient.GetActiveEKSCluster(clusterName) if err != nil { return "", errors.Wrap(err, "failed to get EKS cluster")