Merge pull request #124 from srinandan/export

Export and import connections in a region

Author: srinandan

client/connections/connectors.go

@@ -16,11 +16,13 @@ package connections
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -30,6 +32,24 @@ import (
 	"github.com/srinandan/integrationcli/secmgr"
 )
 
+const maxPageSize = 1000
+
+type listconnections struct {
+	Connections   []connection `json:"connections,omitempty"`
+	NextPageToken string       `json:"nextPageToken,omitempty"`
+}
+
+type connection struct {
+	Name              *string           `json:"name,omitempty"`
+	Description       string            `json:"description,omitempty"`
+	ConnectorVersion  *string           `json:"connectorVersion,omitempty"`
+	ConnectorDetails  *connectorDetails `json:"connectorDetails,omitempty"`
+	ConfigVariables   []configVar       `json:"configVariables,omitempty"`
+	AuthConfig        authConfig        `json:"authConfig,omitempty"`
+	DestinationConfig destinationConfig `json:"destinationConfig,omitempty"`
+	Suspended         bool              `json:"suspended,omitempty"`
+}
+
 type connectionRequest struct {
 	Labels             *map[string]string   `json:"labels,omitempty"`
 	Description        *string              `json:"description,omitempty"`
@@ -375,3 +395,104 @@ func readSecretFile(name string) (payload []byte, err error) {
 	}
 	return content, nil
 }
+
+// Import
+func Import(folder string) (err error) {
+
+	apiclient.SetPrintOutput(false)
+	errs := []string{}
+
+	err = filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			clilog.Warning.Println("connection folder not found")
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if filepath.Ext(path) != ".json" {
+			return nil
+		}
+		name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(filepath.Base(path)))
+		content, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		if _, err := Get(name, ""); err != nil { //create only if connection doesn't exist
+			_, err = Create(name, content, "", "", "", false)
+			if err != nil {
+				errs = append(errs, err.Error())
+			}
+			fmt.Printf("creating connection %s\n", name)
+		} else {
+			fmt.Printf("connection %s already exists, skipping creations\n", name)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil
+	}
+
+	if len(errs) > 0 {
+		return errors.New(strings.Join(errs, "\n"))
+	}
+	return nil
+}
+
+// Export
+func Export(folder string) (err error) {
+	apiclient.SetExportToFile(folder)
+	apiclient.SetPrintOutput(false)
+
+	respBody, err := List(maxPageSize, "", "", "")
+	if err != nil {
+		return err
+	}
+
+	lconnections := listconnections{}
+
+	if err = json.Unmarshal(respBody, &lconnections); err != nil {
+		return err
+	}
+
+	//no connections where found
+	if len(lconnections.Connections) == 0 {
+		return nil
+	}
+
+	for _, lconnection := range lconnections.Connections {
+		lconnection.ConnectorDetails = new(connectorDetails)
+		lconnection.ConnectorDetails.Name = getConnectorName(*lconnection.ConnectorVersion)
+		lconnection.ConnectorDetails.Version = getConnectorVersion(*lconnection.ConnectorVersion)
+		lconnection.ConnectorVersion = nil
+		fileName := getConnectionName(*lconnection.Name) + ".json"
+		lconnection.Name = nil
+		connectionPayload, err := json.Marshal(lconnection)
+		if err != nil {
+			return err
+		}
+		if err = apiclient.WriteByteArrayToFile(path.Join(apiclient.GetExportToFile(), fileName), false, connectionPayload); err != nil {
+			clilog.Error.Println(err)
+			return err
+		}
+		fmt.Printf("Downloaded %s\n", fileName)
+	}
+
+	return nil
+}
+
+func getConnectorName(version string) string {
+	return strings.Split(version, "/")[7]
+}
+
+func getConnectorVersion(version string) int {
+	i, _ := strconv.Atoi(strings.Split(version, "/")[9])
+	return i
+}
+
+func getConnectionName(name string) string {
+	return name[strings.LastIndex(name, "/")+1:]
+}

cmd/authconfigs/export.go

@@ -21,7 +21,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-//ExportCmd to export integrations
+// ExportCmd to export integrations
 var ExportCmd = &cobra.Command{
 	Use:   "export",
 	Short: "Export authconfigs in a region to a folder",
@@ -45,7 +45,7 @@ var folder string
 
 func init() {
 	ExportCmd.Flags().StringVarP(&folder, "folder", "f",
-		"", "Folder to export Integration flows")
+		"", "Folder to export authconfig")
 
 	_ = ExportCmd.MarkFlagRequired("folder")
 }

cmd/connectors/connectors.go

@@ -41,4 +41,6 @@ func init() {
 	Cmd.AddCommand(GetCmd)
 	Cmd.AddCommand(IamCmd)
 	Cmd.AddCommand(NodeCountCmd)
+	Cmd.AddCommand(ExportCmd)
+	Cmd.AddCommand(ImportCmd)
 }

cmd/connectors/export.go

@@ -0,0 +1,51 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectors
+
+import (
+	"github.com/srinandan/integrationcli/apiclient"
+	"github.com/srinandan/integrationcli/client/connections"
+
+	"github.com/spf13/cobra"
+)
+
+// ExportCmd to export connections
+var ExportCmd = &cobra.Command{
+	Use:   "export",
+	Short: "Export connections in a region to a folder",
+	Long:  "Export connections in a region to a folder",
+	Args: func(cmd *cobra.Command, args []string) (err error) {
+		if err = apiclient.SetRegion(region); err != nil {
+			return err
+		}
+		return apiclient.SetProjectID(project)
+	},
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		if err = apiclient.FolderExists(folder); err != nil {
+			return err
+		}
+
+		return connections.Export(folder)
+	},
+}
+
+var folder string
+
+func init() {
+	ExportCmd.Flags().StringVarP(&folder, "folder", "f",
+		"", "Folder to export connections")
+
+	_ = ExportCmd.MarkFlagRequired("folder")
+}

cmd/connectors/import.go

@@ -0,0 +1,49 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectors
+
+import (
+	"github.com/srinandan/integrationcli/apiclient"
+	"github.com/srinandan/integrationcli/client/connections"
+
+	"github.com/spf13/cobra"
+)
+
+// ImportCmd to export connections
+var ImportCmd = &cobra.Command{
+	Use:   "import",
+	Short: "Import connections to a region from a folder",
+	Long:  "Import connections to a region from a folder",
+	Args: func(cmd *cobra.Command, args []string) (err error) {
+		if err = apiclient.SetRegion(region); err != nil {
+			return err
+		}
+		return apiclient.SetProjectID(project)
+	},
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		if err = apiclient.FolderExists(folder); err != nil {
+			return err
+		}
+
+		return connections.Import(folder)
+	},
+}
+
+func init() {
+	ImportCmd.Flags().StringVarP(&folder, "folder", "f",
+		"", "Folder to import connections")
+
+	_ = ImportCmd.MarkFlagRequired("folder")
+}