Skip to content

Commit dca2ece

Browse files
joshuadeanhallJoshua Hall
joshuadeanhall
authored and
Joshua Hall
committed
Add CICD Pipeline scripts and onboard SNOWFLAKE::DATABASE::DATABASE
1 parent 757d881 commit dca2ece

18 files changed

+549
-48
lines changed

‎Snowflake-Database-Database/cleanup.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
#
3+
# This script runs in the buildspec after testing

‎Snowflake-Database-Database/example_inputs/inputs_1_create.json

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
{
2-
"TPSCode": "...",
3-
"Title": "...",
4-
"CoverSheetIncluded": "...",
5-
"DueDate": "...",
6-
"ApprovalDate": "...",
7-
"Memo": "...",
8-
"SecondCopyOfMemo": "...",
9-
"TestCode": "...",
10-
"Authors": "...",
11-
"Tags": "..."
2+
"Name": "EXAMPLENAME",
3+
"DataRetentionTimeInDays": 90,
4+
"MaxDataExtensionTimeInDays": 10,
5+
"DefaultDdlCollation": "en-ci"
126
}

‎Snowflake-Database-Database/example_inputs/inputs_1_update.json

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
{
2-
"TPSCode": "...",
3-
"Title": "...",
4-
"CoverSheetIncluded": "...",
5-
"DueDate": "...",
6-
"ApprovalDate": "...",
7-
"Memo": "...",
8-
"SecondCopyOfMemo": "...",
9-
"TestCode": "...",
10-
"Authors": "...",
11-
"Tags": "..."
2+
"Name": "EXAMPLENAME",
3+
"DataRetentionTimeInDays": 90,
4+
"MaxDataExtensionTimeInDays": 10,
5+
"DefaultDdlCollation": "en-ci"
126
}

‎Snowflake-Database-Database/inputs/inputs_1_create.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"Name": "ExampleName",
2+
"Name": "EXAMPLENAME",
33
"DataRetentionTimeInDays": 90,
44
"MaxDataExtensionTimeInDays": 10,
55
"DefaultDdlCollation": "en-ci"

‎Snowflake-Database-Database/inputs/inputs_1_invalid.json

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
{
2-
"TPSCode": "...",
3-
"Title": "...",
4-
"CoverSheetIncluded": "...",
5-
"DueDate": "...",
6-
"ApprovalDate": "...",
7-
"Memo": "...",
8-
"SecondCopyOfMemo": "...",
9-
"TestCode": "...",
10-
"Authors": "...",
11-
"Tags": "..."
2+
"Name": "DoesNotExists",
3+
"DataRetentionTimeInDays": 1,
4+
"MaxDataExtensionTimeInDays": 3,
5+
"DefaultDdlCollation": "en-ci"
126
}

‎Snowflake-Database-Database/inputs/inputs_1_update.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"Name": "ExampleName",
2+
"Name": "EXAMPLENAME",
33
"DataRetentionTimeInDays": 90,
44
"MaxDataExtensionTimeInDays": 10,
55
"DefaultDdlCollation": "en-ci"

‎Snowflake-Database-Database/overrides.json

-16
This file was deleted.

‎Snowflake-Database-Database/resource-role-prod.yaml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
AWSTemplateFormatVersion: "2010-09-09"
2+
Description: >
3+
This CloudFormation template creates a role assumed by CloudFormation
4+
during CRUDL operations to mutate resources on behalf of the customer.
5+
6+
Resources:
7+
ExecutionRole:
8+
Type: AWS::IAM::Role
9+
Properties:
10+
MaxSessionDuration: 8400
11+
AssumeRolePolicyDocument:
12+
Version: '2012-10-17'
13+
Statement:
14+
- Effect: Allow
15+
Principal:
16+
Service: resources.cloudformation.amazonaws.com
17+
Action: sts:AssumeRole
18+
Path: "/"
19+
Policies:
20+
- PolicyName: ResourceTypePolicy
21+
PolicyDocument:
22+
Version: '2012-10-17'
23+
Statement:
24+
- Effect: Deny
25+
Action:
26+
- "*"
27+
Resource: "*"
28+
Outputs:
29+
ExecutionRoleArn:
30+
Value:
31+
Fn::GetAtt: ExecutionRole.Arn

‎Snowflake-Database-Database/setup.sh

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
#
3+
# This script runs in the buildspec before testing
4+

‎Snowflake-Database-Database/test/integ.yml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Resources:
2+
TestSnowflakeDatabaseDatabase:
3+
Type: Snowflake::Database::Database
4+
Properties:
5+
Name: "TESTSNOWFLAKEDB"
6+
DataRetentionTimeInDays: 90
7+
MaxDataExtensionTimeInDays: 10
8+
DefaultDdlCollation: "en-ci"

‎alpha-buildspec.yml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
version: 0.2
2+
3+
phases:
4+
install:
5+
runtime-versions:
6+
python: 3.7
7+
commands:
8+
- echo Entered the install phase...
9+
- echo About to build $RESOURCE_PATH
10+
- export PATH="/usr/local/bin:$PATH"
11+
- /usr/local/bin/dockerd-entrypoint.sh
12+
- cat /var/log/docker.log
13+
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
14+
- export NVM_DIR="$HOME/.nvm"
15+
- echo "Loading nvm" && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && nvm install 14 && nvm install-latest-npm
16+
- npm --version
17+
- node --version
18+
- pwd
19+
- ls
20+
21+
build:
22+
commands:
23+
- echo Entered the build phase...
24+
- npm install && npm run build
25+
- cd $RESOURCE_PATH
26+
- TYPE_NAME=$(cat .rpdk-config | jq -r .typeName)
27+
- echo "TYPE_NAME is $TYPE_NAME"
28+
- if test -x ./setup.sh; then ./setup.sh; fi
29+
- cfn validate
30+
- cfn generate
31+
- if test -e ./test/setup.yml; then SETUP_STACK_NAME="setup-$(echo $RESOURCE_PATH | sed s/_/-/g)"; fi
32+
- if test -e ./test/setup.yml; then aws cloudformation create-stack --stack-name $SETUP_STACK_NAME --template-body file://test/setup.yml; fi
33+
- if test -e ./test/setup.yml; then aws cloudformation wait stack-create-complete --stack-name $SETUP_STACK_NAME; fi
34+
- npm install && npm run build
35+
- python ../get_type_configuration.py
36+
- cfn submit --dry-run
37+
- nohup sam local start-lambda --log-file sam_output.log &>/dev/null &
38+
- sleep 10
39+
- if ! cfn test --region $AWS_REGION -- -o log_cli=True -o log_cli_level=WARN -o console_output_style=count; then cat sam_output.log && exit 1; fi
40+
finally:
41+
- cat rpdk.log
42+
- if test -e ./test/setup.yml; then aws cloudformation delete-stack --stack-name $SETUP_STACK_NAME; fi
43+
- if test -x ./cleanup.sh; then ./cleanup.sh; fi
44+
artifacts:
45+
files:
46+
- $BUILD_FILE_NAME
47+
name: extensions-build
48+

‎beta-buildspec.yml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
version: 0.2
2+
3+
phases:
4+
install:
5+
runtime-versions:
6+
python: 3.7
7+
commands:
8+
- echo Entered the install phase...
9+
- echo About to build $RESOURCE_PATH
10+
- export PATH="/usr/local/bin:$PATH"
11+
- /usr/local/bin/dockerd-entrypoint.sh
12+
- cat /var/log/docker.log
13+
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
14+
- export NVM_DIR="$HOME/.nvm"
15+
- echo "Loading nvm" && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && nvm install 14 && nvm install-latest-npm
16+
- npm --version
17+
- node --version
18+
- pwd
19+
- ls
20+
21+
build:
22+
commands:
23+
- echo Entered the build phase...
24+
- npm install && npm run build
25+
- cd $RESOURCE_PATH
26+
- TYPE_NAME=$(cat .rpdk-config | jq -r .typeName)
27+
- if test -x ./setup.sh; then ./setup.sh; fi
28+
- cfn validate
29+
- cfn generate
30+
- npm install && npm run build
31+
- cfn submit --region $AWS_REGION --set-default
32+
- python ../get_type_configuration.py
33+
- aws cloudformation set-type-configuration --type RESOURCE --type-name $TYPE_NAME --configuration-alias default --configuration $(cat /root/.cfn-cli/typeConfiguration.json| jq -c "")
34+
- INTEG_STACK_NAME="integ-$(echo $RESOURCE_PATH | tr '[:upper:]' '[:lower:]')"
35+
- aws cloudformation create-stack --stack-name $INTEG_STACK_NAME --template-body file://test/integ.yml
36+
- aws cloudformation wait stack-create-complete --stack-name $INTEG_STACK_NAME
37+
finally:
38+
- cat rpdk.log
39+
- aws cloudformation delete-stack --stack-name $INTEG_STACK_NAME
40+
- if test -x ./cleanup.sh; then ./cleanup.sh; fi
41+
artifacts:
42+
files:
43+
- $BUILD_FILE_NAME
44+
name: extensions-build
45+

‎deregister-all.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
#
3+
# Deregister all private versions of a resource in a region.
4+
#
5+
# Does not un-publish any published versions.
6+
#
7+
# Run this from the resource directory, for example `Snowflake-Database-Database/`
8+
#
9+
# Args
10+
#
11+
# $1 Region
12+
13+
AWS_REGION=$1
14+
15+
TYPE_NAME=$(cat .rpdk-config | jq -r .typeName)
16+
17+
echo "About to deregister all private versions in $AWS_REGION for $TYPE_NAME"
18+
19+
# Iterate over all versions and deregister them
20+
aws cloudformation --region $AWS_REGION list-type-versions --type RESOURCE --type-name $TYPE_NAME | jq '.TypeVersionSummaries[] | .Arn' | xargs -n1 aws cloudformation --region $AWS_REGION deregister-type --arn
21+
22+
# The above will fail for the default version
23+
aws cloudformation --region $AWS_REGION deregister-type --type RESOURCE --type-name $TYPE_NAME || true

‎get_type_configuration.py

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
"Get the Snowflake secret from secrets manager and write it to ~/.cfn-cli/typeConfiguration.json. .rpdk-config files indicates what type configuration the contract tests are looking for"
2+
3+
import boto3
4+
import base64
5+
import os
6+
import pathlib
7+
from botocore.exceptions import ClientError
8+
9+
def get_secret():
10+
"Get the secret from secrets manager"
11+
12+
secret_name = "snowflake-type-configuration"
13+
region_name = "us-east-1"
14+
session = boto3.session.Session()
15+
client = session.client(
16+
service_name="secretsmanager",
17+
region_name=region_name
18+
)
19+
get_secret_value_response = client.get_secret_value(
20+
SecretId=secret_name
21+
)
22+
secret = get_secret_value_response["SecretString"]
23+
home_dir = pathlib.Path.home()
24+
config_dir = os.path.join(home_dir, ".cfn-cli")
25+
26+
if not os.path.exists(config_dir):
27+
os.makedirs(config_dir)
28+
29+
full_path = os.path.join(config_dir, "typeConfiguration.json")
30+
with open(full_path, "w") as f:
31+
f.write(secret)
32+
33+
print(full_path)
34+
35+
36+
if __name__ == "__main__":
37+
get_secret()
38+

‎prod-buildspec.yml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
version: 0.2
2+
3+
phases:
4+
install:
5+
runtime-versions:
6+
python: 3.7
7+
commands:
8+
- echo Entered the install phase...
9+
- echo About to build $RESOURCE_PATH
10+
- export PATH="/usr/local/bin:$PATH"
11+
- /usr/local/bin/dockerd-entrypoint.sh
12+
- cat /var/log/docker.log
13+
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
14+
- export NVM_DIR="$HOME/.nvm"
15+
- echo "Loading nvm" && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && nvm install 14 && nvm install-latest-npm
16+
- npm --version
17+
- node --version
18+
- pwd
19+
- ls
20+
21+
build:
22+
commands:
23+
- echo Entered the build phase...
24+
- npm install && npm run build
25+
- cd $RESOURCE_PATH
26+
- pwd
27+
- ls -l
28+
- ../publish-resource-regions.sh
29+
finally:
30+
- cat rpdk.log
31+
- ./cleanup.sh

‎publish-resource-regions.sh

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/bin/bash
2+
#
3+
# Publish a resource to all regions
4+
#
5+
# Run this from the resource folder
6+
7+
regions_to_publish=(us-east-1 af-south-1 ap-east-1 ap-northeast-1 ap-northeast-2 ap-northeast-3 ap-south-1 ap-southeast-1 ap-southeast-2 ap-southeast-3 ca-central-1 eu-central-1 eu-north-1 eu-south-1 eu-west-1 eu-west-2 eu-west-3 me-central-1 me-south-1 sa-east-1 us-east-2 us-west-1 us-west-2 eu-central-2 eu-south-2 ap-south-2)
8+
9+
# Use this to test succeed-fail locally
10+
#regions_to_publish=(us-east-1 us-west-2 seattle)
11+
12+
successes=()
13+
failures=()
14+
15+
cfn validate
16+
cfn generate
17+
18+
# Create the package
19+
echo "About to run cfn submit --dry-run to create the package"
20+
echo ""
21+
cfn submit --dry-run
22+
echo ""
23+
24+
TYPE_NAME=$(cat .rpdk-config | jq -r .typeName)
25+
TYPE_NAME_LOWER="$(echo $TYPE_NAME | sed s/::/-/g | tr '[:upper:]' '[:lower:]')"
26+
# For example, awscommunity-s3-deletebucketcontents
27+
echo "TYPE_NAME_LOWER is $TYPE_NAME_LOWER"
28+
29+
ZIPFILE="${TYPE_NAME_LOWER}.zip"
30+
echo "ZIPFILE is $ZIPFILE"
31+
32+
ACCOUNT_ID=$(aws sts get-caller-identity|jq -r .Account)
33+
echo "ACCOUNT_ID is $ACCOUNT_ID"
34+
35+
HANDLER_BUCKET="cep-handler-${ACCOUNT_ID}"
36+
37+
# We only need to copy the handler zip once, since it's not regional
38+
# We use this bucket for logs also. The only drawback is if there are
39+
# failures, it can be hard to sort out which log file belongs to which region.
40+
41+
echo "Copying schema package handler to $HANDLER_BUCKET"
42+
aws s3 cp $ZIPFILE s3://$HANDLER_BUCKET/$ZIPFILE
43+
44+
for region in ${regions_to_publish[@]}
45+
do
46+
echo "About to start publishing to $region"
47+
48+
../deregister-all.sh $region RESOURCE
49+
../publish-resource.sh $region
50+
51+
if [ "$?" -eq 0 ]
52+
then
53+
echo "Publishing to $region succeeded"
54+
successes+=($region)
55+
else
56+
echo "Publishing to $region failed"
57+
failures+=($region)
58+
fi
59+
60+
done
61+
62+
echo "The following regions succeeded:"
63+
for s in "${successes[@]}"
64+
do
65+
echo $s
66+
done
67+
68+
echo "The following regions failed:"
69+
for f in "${failures[@]}"
70+
do
71+
echo $f
72+
done

0 commit comments

Comments
 (0)