Skip to content
This repository has been archived by the owner on Mar 9, 2023. It is now read-only.

Commit

Permalink
Merge pull request #91 from srinandan/issue88
Browse files Browse the repository at this point in the history 
connection create improvements
  • Loading branch information
@srinandan
srinandan authored Jan 15, 2023
2 parents 7bb1311 + de76df9 commit 2d02e92
Show file tree
Hide file tree
Showing 17 changed files with 306 additions and 112 deletions.
35 changes: 31 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,26 +114,34 @@ Google managed applications include systems like BigQuery, PubSub, Cloud SQL etc
"configVariables": [ ## these values are specific to each connector type. this example is for pubsub
{
"key": "project_id",
"stringValue": "your-project-id" ## replace this
"stringValue": "$PROJECT_ID" ## if the project id is the same as the connection, use the variable. Otherwise set the project id explicitly
},
{
"key": "topic_id",
"stringValue": "mytopic"
}
],
"serviceAccount": "[email protected]" ## replace this with a SA that has access to the application
]
}
```

NOTE: For ConfigVariables that take a region (ex: CloudSQL), you can also use `$REGION$`

Then execute via `integrationcli` like this:

```sh
integrationcli connectors create -n name-of-the-connector -f ./test/pub_sub_connection.json
```

You can optionally pass the service account to be used from the command line:

```sh
integrationcli connectors create -n name-of-the-connector -f ./test/pub_sub_connection.json -sa <sa-name> -sp <sa-project-id>
```

**NOTES:**

* This command assumes the token is cached, otherwise pass the token via `-t`
* If the service account project is not passed and the service account name is passed, then the connection's project id is used
* If the service account doesn't exist, it will be created
* For PubSub & BigQuery and GCS `integrationcli` adds the IAM permissions for the service account to the resource

Expand Down Expand Up @@ -164,7 +172,7 @@ Third party application include connectors like Salesforce, Service Now, etc. To
"username": "demo",
"passwordDetails": {
"secretName": "sftp-demo", ## this secret is provisioned if it doesn't already exist
"reference": "./test/password.txt" ## this file contains the data/contents to put in secret manager
"reference": "./test/password.txt" ## this file contains the data/contents (encrypted or clear) to put in secret manager
}
}
}
Expand All @@ -181,6 +189,24 @@ integrationcli connectors create -n name-of-the-connector -f ./test/sftp_connect

NOTE: This command assumes the token is cached, otherwise pass the token via `-t`

### Encrypting the Password

When setting the `passwordDetails`, the contents of the password can be encrypted using Cloud KMS

```json
"passwordDetails": {
"secretName": "sftp-demo",
"reference": "./test/password.txt" ## the file containing the password - clear text or encrypted
}
```

The file for the password can be in clear text or encrypted text. If encrypted, then a cloud kms key can be passed for decryption. Before storing the file, the file can be encrypted like this:

```sh
gcloud kms encrypt --plaintext-file=./test/password.txt --keyring $key-ring --project $project --location us-west1 --ciphertext-file=enc_passsword.txt --key=$key
base64 ./test/enc_password.txt > ./test/b64_enc_password.txt # on MacOS, use base64 -i ./test/enc_password.txt > ./test/b64_enc_password.txt
```

### Examples of Creating Connectors

* [Big Query](./test/bq_connection.json)
Expand All @@ -189,6 +215,7 @@ NOTE: This command assumes the token is cached, otherwise pass the token via `-t
* [Salesfoce with JWT](./test/salesforce_jwt_connection.json)
* [Oracle](./test/oracle_connection.json)
* [GCS](./test/gcs_connection.json)
* [CloudSQL - MySQL](./test/cloudsql_mysql_connection.json)

___

Expand Down
209 changes: 118 additions & 91 deletions apiclient/iam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,39 +119,6 @@ func iamServiceAccountExists(iamname string) (code int, err error) {
}
}

func CreateServiceAccount(iamname string) (err error) {

var statusCode int

projectid, displayname, err := getNameAndProject(iamname)
if err != nil {
return err
}

if statusCode, err = iamServiceAccountExists(iamname); err != nil {
return err
}

switch statusCode {
case 200:
return nil
case 404:
var createendpoint = fmt.Sprintf("https://iam.googleapis.com/v1/projects/%s/serviceAccounts", projectid)
iamPayload := []string{}
iamPayload = append(iamPayload, "\"accountId\":\""+displayname+"\"")
iamPayload = append(iamPayload, "\"serviceAccount\": {\"displayName\": \""+displayname+"\"}")
payload := "{" + strings.Join(iamPayload, ",") + "}"

if _, err = HttpClient(false, createendpoint, payload); err != nil {
clilog.Error.Println(err)
return err
}
return nil
default:
return fmt.Errorf("unable to fetch service account details, err: %d", statusCode)
}
}

// setIAMPermission set permissions for a member
func setIAMPermission(endpoint string, name string, memberName string, role string, memberType string) (err error) {

Expand Down Expand Up @@ -205,6 +172,104 @@ func setIAMPermission(endpoint string, name string, memberName string, role stri
return err
}

// setProjectIAMPermission
func setProjectIAMPermission(project string, memberName string, role string) (err error) {
var getendpoint = fmt.Sprintf("https://cloudresourcemanager.googleapis.com/v1/projects/%s:getIamPolicy", project)
var setendpoint = fmt.Sprintf("https://cloudresourcemanager.googleapis.com/v1/projects/%s:setIamPolicy", project)

//this method treats errors as info since this is not a blocking problem

//Get the current IAM policies for the project
respBody, err := HttpClient(false, getendpoint, "")
if err != nil {
clilog.Info.Printf("error getting IAM policies for the project %s: %v", project, err)
return err
}

//binding for IAM Roles
type roleBinding struct {
Role string `json:"role,omitempty"`
Members []string `json:"members,omitempty"`
Condition *condition `json:"condition,omitempty"`
}

// IamPolicy holds the response
type iamPolicy struct {
Version int `json:"version,omitempty"`
Etag string `json:"etag,omitempty"`
Bindings []roleBinding `json:"bindings,omitempty"`
}

//iamPolicyRequest holds the request to set IAM
type iamPolicyRequest struct {
Policy iamPolicy `json:"policy,omitempty"`
}

policy := iamPolicy{}

err = json.Unmarshal(respBody, &policy)
if err != nil {
clilog.Info.Println(err)
return err
}

binding := roleBinding{}
binding.Role = role
binding.Members = append(binding.Members, "serviceAccount:"+memberName)

policy.Bindings = append(policy.Bindings, binding)

policyRequest := iamPolicyRequest{}
policyRequest.Policy = policy
policyRequestBody, err := json.Marshal(policyRequest)
if err != nil {
clilog.Info.Println(err)
return err
}

_, err = HttpClient(false, setendpoint, string(policyRequestBody))
if err != nil {
clilog.Info.Printf("error setting IAM policies for the project %s: %v", project, err)
return err
}

return nil
}

// CreateServiceAccount
func CreateServiceAccount(iamname string) (err error) {

var statusCode int

projectid, displayname, err := getNameAndProject(iamname)
if err != nil {
return err
}

if statusCode, err = iamServiceAccountExists(iamname); err != nil {
return err
}

switch statusCode {
case 200:
return nil
case 404:
var createendpoint = fmt.Sprintf("https://iam.googleapis.com/v1/projects/%s/serviceAccounts", projectid)
iamPayload := []string{}
iamPayload = append(iamPayload, "\"accountId\":\""+displayname+"\"")
iamPayload = append(iamPayload, "\"serviceAccount\": {\"displayName\": \""+displayname+"\"}")
payload := "{" + strings.Join(iamPayload, ",") + "}"

if _, err = HttpClient(false, createendpoint, payload); err != nil {
clilog.Error.Println(err)
return err
}
return nil
default:
return fmt.Errorf("unable to fetch service account details, err: %d", statusCode)
}
}

// SetConnectorIAMPermission set permissions for a member on a connection
func SetConnectorIAMPermission(name string, memberName string, iamRole string, memberType string) (err error) {
var role string
Expand Down Expand Up @@ -236,6 +301,19 @@ func SetPubSubIAMPermission(project string, topic string, memberName string) (er
return setIAMPermission(endpoint, topic, memberName, role, memberType)
}

// SetSecretManagerIAMPermission set permissions for a SA on a secret
func SetSecretManagerIAMPermission(project string, secretName string, memberName string) (err error) {
var endpoint = fmt.Sprintf("https://secretmanager.googleapis.com/v1/projects/%s/secrets", project)
const memberType = "serviceAccount"
const role1 = "roles/secretmanager.secretAccessor"
const role2 = "roles/secretmanager.viewer"
if err = setIAMPermission(endpoint, secretName, memberName, role1, memberType); err != nil {
return err
}
return setIAMPermission(endpoint, secretName, memberName, role2, memberType)
}

// SetBigQueryIAMPermission
func SetBigQueryIAMPermission(project string, datasetid string, memberName string) (err error) {
var endpoint = fmt.Sprintf("https://bigquery.googleapis.com/bigquery/v2/projects/%s/datasets/%s", project, datasetid)
const role = "WRITER"
Expand Down Expand Up @@ -284,69 +362,18 @@ func SetBigQueryIAMPermission(project string, datasetid string, memberName strin
return nil
}

// SetCloudStorageIAMPermission
func SetCloudStorageIAMPermission(project string, memberName string) (err error) {
var getendpoint = fmt.Sprintf("https://cloudresourcemanager.googleapis.com/v1/projects/%s:getIamPolicy", project)
var setendpoint = fmt.Sprintf("https://cloudresourcemanager.googleapis.com/v1/projects/%s:setIamPolicy", project)
//the connector currently requires storage.buckets.list. other built-in roles didn't have this permission
const role = "roles/storage.admin"

//this method treats errors as info since this is not a blocking problem

//Get the current IAM policies for the project
respBody, err := HttpClient(false, getendpoint, "")
if err != nil {
clilog.Info.Printf("error getting IAM policies for the project %s: %v", project, err)
return err
}

//binding for IAM Roles
type roleBinding struct {
Role string `json:"role,omitempty"`
Members []string `json:"members,omitempty"`
Condition *condition `json:"condition,omitempty"`
}

// IamPolicy holds the response
type iamPolicy struct {
Version int `json:"version,omitempty"`
Etag string `json:"etag,omitempty"`
Bindings []roleBinding `json:"bindings,omitempty"`
}

//iamPolicyRequest holds the request to set IAM
type iamPolicyRequest struct {
Policy iamPolicy `json:"policy,omitempty"`
}

policy := iamPolicy{}

err = json.Unmarshal(respBody, &policy)
if err != nil {
clilog.Info.Println(err)
return err
}

binding := roleBinding{}
binding.Role = role
binding.Members = append(binding.Members, "serviceAccount:"+memberName)

policy.Bindings = append(policy.Bindings, binding)

policyRequest := iamPolicyRequest{}
policyRequest.Policy = policy
policyRequestBody, err := json.Marshal(policyRequest)
if err != nil {
clilog.Info.Println(err)
return err
}

_, err = HttpClient(false, setendpoint, string(policyRequestBody))
if err != nil {
clilog.Info.Printf("error setting IAM policies for the project %s: %v", project, err)
return err
}
return setProjectIAMPermission(project, memberName, role)
}

return nil
// SetCloudSQLIAMPermission
func SetCloudSQLIAMPermission(project string, memberName string) (err error) {
const role = "roles/cloudsql.editor"
return setProjectIAMPermission(project, memberName, role)
}

func getNameAndProject(iamFullName string) (projectid string, name string, err error) {
Expand Down
14 changes: 14 additions & 0 deletions cicd/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,20 @@ Grant the Application Integration Admin role to the Cloud Build Service Agent
--role="roles/integrations.integrationAdmin"
```

## Recommended Folder Structure

```bash
├── cloudbuild.yaml #the cloud build deployment file
├── connectors
│   └── <connector-name>.json #there is one file per connector. the connector name is the file name.
├── authconfig
│   └── <authconfig-name>.json #there is one file per authconfig. the authconfig name is the file name.
├── overrides
│   └── overrides.json #always name this overrides.json. there is only one file in this folder
└── src
└── <integration-name>.json #there only one file in the folder. the integration name is the file name.
```

## Steps

1. Download the integration from the UI or using `integrationcli`. Here is an example to download via CLI:
Expand Down
Loading

0 comments on commit 2d02e92

Please sign in to comment.