handle sdfc and connectors #134

Author: srinandan

.github/workflows/docker-cloudbuild.yml

@@ -1,3 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: Cloud Builder Docker
 
 # This workflow uses actions that are not certified by GitHub.

.github/workflows/docker-publish.yml

@@ -1,3 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: Docker
 
 # This workflow uses actions that are not certified by GitHub.

Dockerfile

@@ -1,3 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM golang:1.19 as builder
 
 ADD ./apiclient /go/src/integrationcli/apiclient

cicd/README.md

@@ -44,42 +44,17 @@ Grant the Application Integration Admin role to the Cloud Build Service Agent
 
 ## Steps
 
-1. Download the integration from the UI or using `integrationcli`. Here is an example to download via CLI:
+1. Generate a scaffolding:
 
 ```sh
 
 token=$(gcloud auth print-access-token)
-integrationcli integrations versions get -n <integration-name> -v <version> -p <project-id> -r <region-name> -t $token > ./src/<integration-name>.json
+integrationcli integrations scaffolding -n <integration-name> -s <snapShot> -p <project-id> -r <region-name> -t $token
 ```
 
-You can also download via a snapshot number like this:
+Inspect the generated `overrides`, `connectors` and `authconfigs`
 
-```sh
-
-token=$(gcloud auth print-access-token)
-integrationcli integrations versions get -n <integration-name> -s <snapshot> -p <dev-project-id> -r <region-name> -t $token > ./src/<integration-name>.json
-```
-
-2. Author overrides (specific for the environment) and store them in the overrides folder. Here is an example overrides for the URL in the REST task
-
-```json
-{
-    "task_overrides": [{
-        "taskId": "1",
-        "task:": "GenericRestV2Task",
-        "parameters":  {
-            "url": {
-                "key": "url",
-                "value": {
-                    "stringValue": "https://httpbin.org/ip"
-                }
-            }
-        }
-    }]
-}
-```
-
-3. Trigger the build manually
+2. Trigger the build manually
 
 ```sh
 

client/connections/connectors.go

@@ -150,7 +150,7 @@ type nodeConfig struct {
 }
 
 // Create
-func Create(name string, content []byte, serviceAccountName string, serviceAccountProject string, encryptionKey string, grantPermission bool) (respBody []byte, err error) {
+func Create(name string, content []byte, serviceAccountName string, serviceAccountProject string, encryptionKey string, grantPermission bool, createSecret bool) (respBody []byte, err error) {
 
 	var secretVersion string
 
@@ -278,34 +278,54 @@ func Create(name string, content []byte, serviceAccountName string, serviceAccou
 	c.ConnectorDetails = nil
 
 	//handle secrets for username
-	if c.AuthConfig != nil && c.AuthConfig.UserPassword.PasswordDetails != nil {
-		payload, err := readSecretFile(c.AuthConfig.UserPassword.PasswordDetails.Reference)
-		if err != nil {
-			return nil, err
-		}
-
-		//check if a Cloud KMS key was passsed, assume the file is encrypted
-		if encryptionKey != "" {
-			encryptionKey := path.Join("projects", apiclient.GetProjectID(), encryptionKey)
-			payload, err = cloudkms.DecryptSymmetric(encryptionKey, payload)
-			if err != nil {
-				return nil, err
+	if c.AuthConfig != nil {
+		switch c.AuthConfig.AuthType {
+		case "USER_PASSWORD":
+			if c.AuthConfig.UserPassword.PasswordDetails != nil {
+				if createSecret {
+					payload, err := readSecretFile(c.AuthConfig.UserPassword.PasswordDetails.Reference)
+					if err != nil {
+						return nil, err
+					}
+
+					//check if a Cloud KMS key was passsed, assume the file is encrypted
+					if encryptionKey != "" {
+						encryptionKey := path.Join("projects", apiclient.GetProjectID(), encryptionKey)
+						payload, err = cloudkms.DecryptSymmetric(encryptionKey, payload)
+						if err != nil {
+							return nil, err
+						}
+					}
+
+					if secretVersion, err = secmgr.Create(apiclient.GetProjectID(), c.AuthConfig.UserPassword.PasswordDetails.SecretName, payload); err != nil {
+						return nil, err
+					}
+
+					secretName := c.AuthConfig.UserPassword.PasswordDetails.SecretName
+					c.AuthConfig.UserPassword.Password = new(secret)
+					c.AuthConfig.UserPassword.Password.SecretVersion = secretVersion
+					c.AuthConfig.UserPassword.PasswordDetails = nil //clean the input
+					if grantPermission && c.ServiceAccount != nil {
+						//grant connector service account access to secretVersion
+						if err = apiclient.SetSecretManagerIAMPermission(apiclient.GetProjectID(), secretName, *c.ServiceAccount); err != nil {
+							return nil, err
+						}
+					}
+				} else {
+					c.AuthConfig.UserPassword.Password = new(secret)
+					c.AuthConfig.UserPassword.Password.SecretVersion = fmt.Sprintf("projects/%s/secrets/%s/versions/1", apiclient.GetProjectID(), c.AuthConfig.UserPassword.PasswordDetails.SecretName)
+					c.AuthConfig.UserPassword.PasswordDetails = nil //clean the input
+				}
 			}
-		}
-
-		if secretVersion, err = secmgr.Create(apiclient.GetProjectID(), c.AuthConfig.UserPassword.PasswordDetails.SecretName, payload); err != nil {
-			return nil, err
-		}
-		secretName := c.AuthConfig.UserPassword.PasswordDetails.SecretName
-		c.AuthConfig.UserPassword.Password = new(secret)
-		c.AuthConfig.UserPassword.Password.SecretVersion = secretVersion
-		c.AuthConfig.UserPassword.PasswordDetails = nil //clean the input
-
-		if grantPermission && c.ServiceAccount != nil {
-			//grant connector service account access to secretVersion
-			if err = apiclient.SetSecretManagerIAMPermission(apiclient.GetProjectID(), secretName, *c.ServiceAccount); err != nil {
-				return nil, err
+		case "OAUTH2_JWT_BEARER":
+			if createSecret {
+				clilog.Warning.Println("Creating secrets for OAUTH2_JET_BEARER is not implemented")
+			} else {
+				c.AuthConfig.Oauth2JwtBearer.ClientKey.SecretVersion = fmt.Sprintf("projects/%s/secrets/%s/versions/1", apiclient.GetProjectID(), c.AuthConfig.Oauth2JwtBearer.ClientKeyDetails.SecretName)
 			}
+		case "OAUTH2_CLIENT_CREDENTIALS":
+		default:
+			clilog.Warning.Printf("Creating secrets for %s is not implemented\n", c.AuthConfig.AuthType)
 		}
 	}
 
@@ -331,7 +351,7 @@ func Delete(name string) (respBody []byte, err error) {
 }
 
 // Get
-func Get(name string, view string, minimal bool) (respBody []byte, err error) {
+func Get(name string, view string, minimal bool, overrides bool) (respBody []byte, err error) {
 	u, _ := url.Parse(apiclient.GetBaseConnectorURL())
 	q := u.Query()
 	if view != "" {
@@ -358,6 +378,27 @@ func Get(name string, view string, minimal bool) (respBody []byte, err error) {
 		c.ConnectorDetails.Version = getConnectorVersion(*c.ConnectorVersion)
 		c.ConnectorVersion = nil
 		c.Name = nil
+		if overrides {
+			switch c.AuthConfig.AuthType {
+			case "USER_PASSWORD":
+				p := c.AuthConfig.UserPassword.Password.SecretVersion
+				c.AuthConfig.UserPassword.PasswordDetails = new(secretDetails)
+				c.AuthConfig.UserPassword.PasswordDetails.SecretName = strings.Split(p, "/")[3]
+				c.AuthConfig.UserPassword.Password = nil
+			case "OAUTH2_JWT_BEARER":
+				p := c.AuthConfig.Oauth2JwtBearer.ClientKey.SecretVersion
+				c.AuthConfig.Oauth2JwtBearer.ClientKeyDetails = new(secretDetails)
+				c.AuthConfig.Oauth2JwtBearer.ClientKeyDetails.SecretName = strings.Split(p, "/")[3]
+				c.AuthConfig.Oauth2JwtBearer.ClientKey = nil
+			}
+			if isGoogleConnection(c.ConnectorDetails.Name) {
+				for _, configVar := range c.ConfigVariables {
+					if configVar.Key == "project_id" {
+						*configVar.StringValue = "$PROJECT_ID$"
+					}
+				}
+			}
+		}
 		connectionPayload, err := json.Marshal(c)
 		if err != nil {
 			return nil, err
@@ -426,7 +467,7 @@ func readSecretFile(name string) (payload []byte, err error) {
 }
 
 // Import
-func Import(folder string) (err error) {
+func Import(folder string, createSecret bool) (err error) {
 
 	apiclient.SetPrintOutput(false)
 	errs := []string{}
@@ -448,8 +489,8 @@ func Import(folder string) (err error) {
 			return err
 		}
 
-		if _, err := Get(name, "", false); err != nil { //create only if connection doesn't exist
-			_, err = Create(name, content, "", "", "", false)
+		if _, err := Get(name, "", false, false); err != nil { //create only if connection doesn't exist
+			_, err = Create(name, content, "", "", "", false, createSecret)
 			if err != nil {
 				errs = append(errs, err.Error())
 			}
@@ -525,3 +566,11 @@ func getConnectorVersion(version string) int {
 func getConnectionName(name string) string {
 	return name[strings.LastIndex(name, "/")+1:]
 }
+
+func isGoogleConnection(connectionName string) bool {
+	if connectionName == "pubsub" || connectionName == "gcs" || connectionName == "biqguery" ||
+		connectionName == "cloudsql-mysql" || connectionName == "cloudsql-postgresql" || connectionName == "cloudsql-sqlserver" {
+		return true
+	}
+	return false
+}

client/integrations/integrations.go

@@ -702,12 +702,38 @@ func GetAuthConfigs(integration []byte) (authconfigs []string, err error) {
 				authConfigUuid := getAuthConfigUuid(*authConfigParams.Value.JsonValue)
 				authconfigs = append(authconfigs, authConfigUuid)
 			}
+		} else if taskConfig.Task == "CloudFunctionTask" {
+			authConfigParams := taskConfig.Parameters["authConfig"]
+			if authConfigParams.Key == "authConfig" {
+				authConfigUuid := getAuthConfigUuid(*authConfigParams.Value.JsonValue)
+				authconfigs = append(authconfigs, authConfigUuid)
+			}
 		}
 	}
 
 	return authconfigs, err
 }
 
+// GetSfdcInstances
+func GetSfdcInstances(integration []byte) (instances map[string]string, err error) {
+	iversion := integrationVersion{}
+
+	err = json.Unmarshal(integration, &iversion)
+	if err != nil {
+		return instances, err
+	}
+
+	instances = make(map[string]string)
+
+	for _, triggerConfig := range iversion.TriggerConfigs {
+		if triggerConfig.TriggerType == "SFDC_CHANNEL" {
+			instances[triggerConfig.Properties["SFDC instance name"]] = triggerConfig.Properties["Channel name"]
+		}
+	}
+
+	return instances, err
+}
+
 // GetConnections
 func GetConnections(integration []byte) (connections []string, err error) {
 	iversion := integrationVersion{}

client/integrations/overrides.go

@@ -199,6 +199,19 @@ func extractOverrides(iversion integrationVersion) (overrides, error) {
 			taskOverrides.ParamOverrides = append(taskOverrides.ParamOverrides, ip)
 		}
 	}
+	for _, triggerConfig := range iversion.TriggerConfigs {
+		if triggerConfig.TriggerType == "CLOUD_PUBSUB_EXTERNAL" {
+			subscription := triggerConfig.Properties["Subscription name"]
+			triggerOverride := triggeroverrides{}
+			triggerOverride.ProjectId = new(string)
+			triggerOverride.TopicName = new(string)
+			*triggerOverride.ProjectId = strings.Split(subscription, "_")[0]
+			*triggerOverride.TopicName = strings.Split(subscription, "_")[1]
+			triggerOverride.TriggerNumber = triggerConfig.TriggerNumber
+			taskOverrides.TriggerOverrides = append(taskOverrides.TriggerOverrides, triggerOverride)
+		}
+	}
+
 	return taskOverrides, nil
 }
 
@@ -302,7 +315,7 @@ func getNewConnectionParams(connectionName string, connectionLocation string) (c
 		integrationRegion = apiclient.GetRegion() //store the integration location
 		apiclient.SetRegion(connectionLocation)   //set the connector region
 	}
-	connResp, err := connections.Get(connectionName, "BASIC", false) //get connector details
+	connResp, err := connections.Get(connectionName, "BASIC", false, false) //get connector details
 	apiclient.SetPrintOutput(true)
 	if connectionLocation != "" {
 		apiclient.SetRegion(integrationRegion) //set the integration region back