diff --git a/Makefile b/Makefile index 0973be3b..25f3be18 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,10 @@ go-mod-tidy: go-test: ./scripts/go-test.sh +.PHONY: check +check: + golangci-lint run + .PHONY: docker-check docker: check docker build -t layer5/meshery-meshsync . diff --git a/go.mod b/go.mod index 53e2f913..f2f64147 100644 --- a/go.mod +++ b/go.mod @@ -15,11 +15,13 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/layer5io/meshery-adapter-library v0.1.9 github.com/layer5io/meshery-operator v0.2.0 - github.com/layer5io/meshkit v0.1.30 + github.com/layer5io/meshkit v0.1.32 github.com/myntra/pipeline v0.0.0-20180618182531-2babf4864ce8 github.com/nats-io/nats.go v1.10.0 google.golang.org/grpc v1.34.0 google.golang.org/protobuf v1.25.0 + gorm.io/driver/sqlite v1.1.4 // indirect + gorm.io/gorm v1.20.10 istio.io/client-go v1.8.1 k8s.io/api v0.18.12 k8s.io/apimachinery v0.18.12 diff --git a/go.sum b/go.sum index 9a633637..faefc08a 100644 --- a/go.sum +++ b/go.sum @@ -598,6 +598,10 @@ github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 h1:7nkB9fLPMwtn github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -662,6 +666,9 @@ github.com/layer5io/meshery-operator v0.2.0 h1:u+p6veNuqe5QznLWnjZHC796p2h2bZvZR github.com/layer5io/meshery-operator v0.2.0/go.mod h1:qlGpHvdsZuwwn048LxCMYUlKYYLJnY3R2JD3/p8v71I= github.com/layer5io/meshkit v0.1.30 h1:d9emBQAf+YnURkA5wZ6VbbqDfnx9G8G3xqOv6dG4n/k= github.com/layer5io/meshkit v0.1.30/go.mod h1:AznOL6xqpUZGyExSZJ3Bdx6EZ22UnAT9V620pm7R484= +github.com/layer5io/meshkit v0.1.31 h1:09kfaoEbxqJ8Rc20XPudUnXoRU2QsKcGRp87ebNd7+E= +github.com/layer5io/meshkit v0.1.32 h1:en4T3bo/BVtO/CsPbWx/1bgObCKtRHa+0tZtoxOWISw= +github.com/layer5io/meshkit v0.1.32/go.mod h1:IkAIh0mwGkd52xLQ6vhpJuip+XUkrImnWnGJ08Lox7Q= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= @@ -711,6 +718,8 @@ github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= +github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1500,6 +1509,13 @@ gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= +gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= +gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.9 h1:M3aIZKXAC1PtPVu9t3WGwkBTE1le5c2telz3I/qjRNg= +gorm.io/gorm v1.20.9/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.10 h1:oJDuZyiUVVH1t20aRv439xIhmx5HFqv3iDxwAZ5sBb0= +gorm.io/gorm v1.20.10/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= helm.sh/helm/v3 v3.3.1 h1:uc+ZUthJnWNSwqyIv1KCdQm0ewi0eAf6oRaWG2X1oo0= diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 11f5ad2f..a24f8273 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -2,43 +2,21 @@ package cache import ( "github.com/google/uuid" - mesherykube "github.com/layer5io/meshkit/utils/kubernetes" - discovery "github.com/layer5io/meshsync/pkg/discovery" + corev1 "k8s.io/api/core/v1" ) var ( - ClusterName string - ClusterID string - Storage map[string][]string + ClusterID string + Namespaces []string ) func init() { - // Initialize Kubeconfig - rconfig, err1 := mesherykube.DetectKubeConfig() - if err1 != nil { - return - } - - // Configure discovery client - dclient, err2 := discovery.NewClient(rconfig) - if err2 != nil { - return - } - - namespaces, err3 := dclient.ListNamespaces() - if err3 != nil { - return - } - var namespacesName []string + Namespaces = make([]string, 0) + ClusterID = uuid.New().String() +} - // processing +func SetNamespaces(namespaces []corev1.Namespace) { for _, namespace := range namespaces { - namespacesName = append(namespacesName, namespace.Name) + Namespaces = append(Namespaces, namespace.Name) } - - Storage = make(map[string][]string) - Storage["NamespaceNames"] = namespacesName - - ClusterName = namespaces[0].ClusterName - ClusterID = uuid.New().String() } diff --git a/internal/cluster/pipeline/deployment.go b/internal/cluster/pipeline/deployment.go index 86da3b64..68fd9e85 100644 --- a/internal/cluster/pipeline/deployment.go +++ b/internal/cluster/pipeline/deployment.go @@ -30,10 +30,7 @@ func (d *Deployment) Exec(request *pipeline.Request) *pipeline.Result { // it will contain a pipeline to run log.Println("Deployment Discovery Started") - // get all namespaces - namespaces := cache.Storage["NamespaceNames"] - - for _, namespace := range namespaces { + for _, namespace := range cache.Namespaces { // get Deployments deployments, err := d.client.ListDeployments(namespace) if err != nil { diff --git a/internal/cluster/pipeline/namespace.go b/internal/cluster/pipeline/namespace.go index 2ad86a11..a77e0bac 100644 --- a/internal/cluster/pipeline/namespace.go +++ b/internal/cluster/pipeline/namespace.go @@ -3,6 +3,7 @@ package pipeline import ( "log" + "github.com/layer5io/meshsync/internal/cache" broker "github.com/layer5io/meshsync/pkg/broker" discovery "github.com/layer5io/meshsync/pkg/discovery" "github.com/layer5io/meshsync/pkg/model" @@ -36,6 +37,7 @@ func (n *Namespace) Exec(request *pipeline.Request) *pipeline.Result { Error: err, } } + cache.SetNamespaces(namespaces) // processing for _, namespace := range namespaces { diff --git a/internal/cluster/pipeline/pipeline.go b/internal/cluster/pipeline/pipeline.go index ae8468ee..82aa17e3 100644 --- a/internal/cluster/pipeline/pipeline.go +++ b/internal/cluster/pipeline/pipeline.go @@ -19,7 +19,7 @@ var ( Concurrent: false, Steps: []pipeline.Step{}, } - Subject = "cluster" + Subject = "meshsync" ) func Initialize(client *discovery.Client, broker broker.Handler) *pipeline.Pipeline { diff --git a/internal/cluster/pipeline/pod.go b/internal/cluster/pipeline/pod.go index 98e8e0c2..45e2d596 100644 --- a/internal/cluster/pipeline/pod.go +++ b/internal/cluster/pipeline/pod.go @@ -30,10 +30,7 @@ func (p *Pod) Exec(request *pipeline.Request) *pipeline.Result { // it will contain a pipeline to run log.Println("Pod Discovery Started") - // get all namespaces - namespaces := cache.Storage["NamespaceNames"] - - for _, namespace := range namespaces { + for _, namespace := range cache.Namespaces { // get Pods pods, err := p.client.ListPods(namespace) if err != nil { diff --git a/internal/cluster/pipeline/service.go b/internal/cluster/pipeline/service.go index d3b88b8e..f32afdbb 100644 --- a/internal/cluster/pipeline/service.go +++ b/internal/cluster/pipeline/service.go @@ -30,10 +30,7 @@ func (d *Service) Exec(request *pipeline.Request) *pipeline.Result { // it will contain a pipeline to run log.Println("Service Discovery Started") - // get all namespaces - namespaces := cache.Storage["NamespaceNames"] - - for _, namespace := range namespaces { + for _, namespace := range cache.Namespaces { // get Services services, err := d.client.ListServices(namespace) if err != nil { diff --git a/internal/meshes/istio/pipeline/pipeline.go b/internal/meshes/istio/pipeline/pipeline.go index 7d820669..65b02003 100644 --- a/internal/meshes/istio/pipeline/pipeline.go +++ b/internal/meshes/istio/pipeline/pipeline.go @@ -22,7 +22,7 @@ var ( // Namespaces : need some solution for this Namespaces = []string{"default", "istio-system"} - Subject = "istio" + Subject = "meshsync" ) func Initialize(client *discovery.Client, broker broker.Handler) *pipeline.Pipeline { diff --git a/main.go b/main.go index 20c9380c..299b1516 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,6 @@ func main() { err = meshsyncHandler.StartDiscovery() if err != nil { log.Error(err) - os.Exit(1) } // Initialize service by running pre-defined tasks diff --git a/pkg/model/model.go b/pkg/model/model.go index 3b7d508a..ec0dfc94 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -1,45 +1,59 @@ package model import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/layer5io/meshkit/database" ) type Object struct { - Resource KubernetesResource - TypeMeta KubernetesResourceTypeMeta - ObjectMeta KubernetesResourceObjectMeta - Spec KubernetesResourceSpec - Status KubernetesResourceStatus + Index Index `json:"index,omitempty"` + TypeMeta ResourceTypeMeta `json:"typemeta,omitempty" gorm:"foreignKey:ResourceTypeMetaID;references:ID"` + ObjectMeta ResourceObjectMeta `json:"metadata,omitempty" gorm:"foreignKey:ResourceObjectMetaID;references:ID"` + Spec ResourceSpec `json:"spec,omitempty" gorm:"foreignKey:ResourceSpecID;references:ID"` + Status ResourceStatus `json:"status,omitempty" gorm:"foreignKey:ResourceStatusID;references:ID"` } -type KubernetesResource struct { - MesheryResourceID string `json:"meshery-resource-id,omitempty"` - ResourceID string `json:"resource-id,omitempty"` - ResourceTypeMetaID string `json:"resource-type-meta-id,omitempty"` - ResourceObjectMetaID string `json:"resource-object-meta-id,omitempty"` - ResourceSpecID string `json:"resource-spec-id,omitempty"` - ResourceStatusID string `json:"resource-status-id,omitempty"` +type Index struct { + database.Model + ResourceID string `json:"resource-id,omitempty"` + TypeMetaID string `json:"type-meta-id,omitempty"` + ObjectMetaID string `json:"object-meta-id,omitempty"` + SpecID string `json:"spec-id,omitempty"` + StatusID string `json:"status-id,omitempty"` } -type KubernetesResourceTypeMeta struct { - metav1.TypeMeta `json:",inline"` - - ResourceTypeMetaID string `json:"resource-type-meta-id,omitempty"` +type ResourceTypeMeta struct { + database.Model + Kind string `json:"kind,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` } -type KubernetesResourceObjectMeta struct { - metav1.ObjectMeta `json:"metadata,omitempty"` - - ResourceObjectMetaID string `json:"resource-object-meta-id,omitempty"` - ClusterID string `json:"cluster-id,omitempty"` +type ResourceObjectMeta struct { + database.Model + Name string `json:"name,omitempty"` + GenerateName string `json:"generateName,omitempty"` + Namespace string `json:"namespace,omitempty"` + SelfLink string `json:"selfLink,omitempty"` + UID string `json:"uid,omitempty"` + ResourceVersion string `json:"resourceVersion,omitempty"` + Generation int64 `json:"generation,omitempty"` + CreationTimestamp string `json:"creationTimestamp,omitempty"` + DeletionTimestamp string `json:"deletionTimestamp,omitempty"` + DeletionGracePeriodSeconds *int64 `json:"deletionGracePeriodSeconds,omitempty"` + Labels string `json:"labels,omitempty" gorm:"type:json"` + Annotations string `json:"annotations,omitempty" gorm:"type:json"` + // OwnerReferences string `json:"ownerReferences,omitempty" gorm:"type:json"` + // Finalizers string `json:"finalizers,omitempty" gorm:"type:json"` + ClusterName string `json:"clusterName,omitempty"` + // ManagedFields string `json:"managedFields,omitempty" gorm:"type:json"` + ClusterID string `json:"cluster-id,omitempty"` } -type KubernetesResourceSpec struct { - ResourceSpecID string `json:"resource-spec-id,omitempty"` - Attribute map[string]interface{} `json:"attribute,omitempty"` +type ResourceSpec struct { + database.Model + Attribute string `json:"attribute,omitempty" gorm:"type:json"` } -type KubernetesResourceStatus struct { - ResourceStatusID string `json:"resource-status-id,omitempty"` - Attribute map[string]interface{} `json:"attribute,omitempty"` +type ResourceStatus struct { + database.Model + Attribute string `json:"attribute,omitempty" gorm:"type:json"` } diff --git a/pkg/model/model_converter.go b/pkg/model/model_converter.go index a24dff3c..7723c34d 100644 --- a/pkg/model/model_converter.go +++ b/pkg/model/model_converter.go @@ -1,74 +1,104 @@ package model import ( - "encoding/json" - "github.com/google/uuid" + "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshsync/internal/cache" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func ConvObject(typeMeta metav1.TypeMeta, objectMeta metav1.ObjectMeta, spec interface{}, status interface{}) Object { - kubernetesResource := getKubernetesResource() - kubernetesResourceTypeMeta := getKubernetesResourceTypeMeta(typeMeta, kubernetesResource.ResourceTypeMetaID) - kubernetesResourceObjectMeta := getKubernetesResourceObjectMeta(objectMeta, kubernetesResource.ResourceObjectMetaID) - kubernetesResourceSpec := getKubernetesResourceSpec(spec, kubernetesResource.ResourceSpecID) - kubernetesResourceStatus := getKubernetesResourceStatus(status, kubernetesResource.ResourceStatusID) + index := generateIndex() + resourceTypeMeta := makeTypeMeta(typeMeta, index.TypeMetaID) + resourceObjectMeta := makeObjectMeta(objectMeta, index.ObjectMetaID) + resourceSpec := makeSpec(spec, index.SpecID) + resourceStatus := makeStatus(status, index.StatusID) return Object{ - Resource: kubernetesResource, - TypeMeta: kubernetesResourceTypeMeta, - ObjectMeta: kubernetesResourceObjectMeta, - Spec: kubernetesResourceSpec, - Status: kubernetesResourceStatus, + Index: index, + TypeMeta: resourceTypeMeta, + ObjectMeta: resourceObjectMeta, + Spec: resourceSpec, + Status: resourceStatus, } } -func getKubernetesResource() KubernetesResource { - return KubernetesResource{ - MesheryResourceID: uuid.New().String(), - ResourceID: uuid.New().String(), - ResourceTypeMetaID: uuid.New().String(), - ResourceObjectMetaID: uuid.New().String(), - ResourceSpecID: uuid.New().String(), - ResourceStatusID: uuid.New().String(), +func generateIndex() Index { + return Index{ + ResourceID: uuid.New().String(), + TypeMetaID: uuid.New().String(), + ObjectMetaID: uuid.New().String(), + SpecID: uuid.New().String(), + StatusID: uuid.New().String(), } } -func getKubernetesResourceTypeMeta(resource metav1.TypeMeta, id string) KubernetesResourceTypeMeta { - return KubernetesResourceTypeMeta{ - TypeMeta: resource, - ResourceTypeMetaID: id, +func makeTypeMeta(resource metav1.TypeMeta, id string) ResourceTypeMeta { + return ResourceTypeMeta{ + Model: database.Model{ + ID: id, + }, + Kind: resource.Kind, + APIVersion: resource.APIVersion, } } -func getKubernetesResourceObjectMeta(resource metav1.ObjectMeta, id string) KubernetesResourceObjectMeta { - return KubernetesResourceObjectMeta{ - ObjectMeta: resource, - ResourceObjectMetaID: id, - ClusterID: cache.ClusterID, +func makeObjectMeta(resource metav1.ObjectMeta, id string) ResourceObjectMeta { + labels, _ := utils.Marshal(resource.Labels) + annotations, _ := utils.Marshal(resource.Annotations) + + var creationTime string + var deletionTime string + if resource.CreationTimestamp.IsZero() { + creationTime = resource.CreationTimestamp.String() + } + if resource.DeletionTimestamp.IsZero() { + deletionTime = resource.DeletionTimestamp.String() + } + + return ResourceObjectMeta{ + Model: database.Model{ + ID: id, + }, + Name: resource.Name, + GenerateName: resource.GenerateName, + Namespace: resource.Namespace, + SelfLink: resource.SelfLink, + UID: string(resource.UID), + ResourceVersion: resource.ResourceVersion, + Generation: resource.Generation, + CreationTimestamp: creationTime, + DeletionTimestamp: deletionTime, + DeletionGracePeriodSeconds: resource.DeletionGracePeriodSeconds, + Labels: labels, + Annotations: annotations, + // OwnerReferences: resource.OwnerReferences, + // Finalizers: resource.Finalizers, + ClusterName: resource.ClusterName, + // ManagedFields: resource.ManagedFields, + ClusterID: cache.ClusterID, } } -func getKubernetesResourceSpec(spec interface{}, id string) KubernetesResourceSpec { - specJSON, _ := json.Marshal(spec) - var specTemp map[string]interface{} - _ = utils.Unmarshal(string(specJSON), &specTemp) +func makeSpec(spec interface{}, id string) ResourceSpec { + specJSON, _ := utils.Marshal(spec) - return KubernetesResourceSpec{ - ResourceSpecID: id, - Attribute: specTemp, + return ResourceSpec{ + Model: database.Model{ + ID: id, + }, + Attribute: string(specJSON), } } -func getKubernetesResourceStatus(status interface{}, id string) KubernetesResourceStatus { - statusJSON, _ := json.Marshal(status) - var statusTemp map[string]interface{} - _ = utils.Unmarshal(string(statusJSON), &statusTemp) +func makeStatus(status interface{}, id string) ResourceStatus { + statusJSON, _ := utils.Marshal(status) - return KubernetesResourceStatus{ - ResourceStatusID: id, - Attribute: statusTemp, + return ResourceStatus{ + Model: database.Model{ + ID: id, + }, + Attribute: string(statusJSON), } }