chore: boostrap, synthesize and parametrize (#3)

* WIP synthesize and parametrize

* chore: add bootstrap code and documentation

* chore: more changes

* chore: revert naming

README.md

@@ -1,10 +1,23 @@
 # Dev cluster on AWS via AWS CDK and Cloudformation
 
+## Steps to build and publish
+
+```
+rm -rf cdk.out
+rm -rf cdk.out
+npx cdk synth
+AWS_REGION=eu-central-1 npx cdk-assets publish -p cdk.out/garden-dev-cluster.assets.json
+rain fmt cdk.out/garden-dev-cluster.template.json > test.yaml
+```
+
+
 ## Inputs
 
 * domain e.g. dev.marketplace.sys.garden
 * hosted zone id
 ```
 aws route53 list-hosted-zones-by-name --dns-name dev.marketplace.sys.garden
 ```
-* user arns or role arn to add to the aws-auth configmap
+* user arns or role arn to add to the aws-auth configmap
+
+

bootstrap/.gitignore

@@ -0,0 +1,2 @@
+venv/
+virtualenv/

bootstrap/README.md

@@ -0,0 +1,25 @@
+# About boostrap
+
+For the serverless functions and other assets, we need an S3 bucket in all regions to work around the limitation described here:
+https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-code.html#cfn-lambda-function-code-s3bucket
+
+This creates a public S3 bucket named `garden-cfn-public-<region>` in every region.
+
+This only needs to be executed once.
+
+# Bucket directory structure
+
+- `garden-cfn-public-<region>/`
+  - `dev-cluster/`: Files related to the dev-cluster CDK stack
+    - `x.x.x/`: semver release directory
+       - `<hash>.{json,zip}`: cdk asset created by synth
+       - `dev-cluster-quickstart.yaml`: CloudFormation stack synthesized from CDK code
+
+# How to run
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip3 install boto3
+python3 bootstrap.py
+```

bootstrap/bootstrap.py

@@ -0,0 +1,46 @@
+"""
+MIT Licensed
+Copyright (c) 2021 superwerker
+
+https://github.com/superwerker/superwerker/blob/078231a/cdk/cdk-bootstrap.py
+"""
+import boto3
+from boto3.session import Session
+import subprocess
+import os
+
+class DeployError(Exception):
+    pass
+
+cfn = boto3.client("cloudformation")
+s = Session()
+regions = [ 
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "ap-south-1",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ca-central-1",
+    "eu-central-1",
+    "eu-north-1",
+    "eu-west-1",
+    "eu-west-2",
+    "eu-west-3",
+    "sa-east-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-2"
+]
+
+template_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bootstrap.yaml")
+for region in regions:
+    cmd = ["aws", "cloudformation", "deploy",
+             "--stack-name", "garden-marketplace-assets-bootstrap",
+             "--region", region,
+             "--template-file", template_file,
+             ]
+    print("Deploying to region={0} with cmd={1}".format(region, cmd))
+    p = subprocess.run(cmd, capture_output=True)
+    if p.returncode != 0:
+        raise DeployError(p.stderr)
+    print(p.stdout)

bootstrap/bootstrap.yaml

@@ -0,0 +1,36 @@
+# MIT Licensed
+# Copyright (c) 2021 superwerker
+# https://github.com/superwerker/superwerker/blob/b05f55e/cdk/cdk-bootstrap.yaml
+Resources:
+  BootStrapBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: !Sub 'garden-cfn-public-${AWS::Region}'
+      PublicAccessBlockConfiguration:
+        # Ignore ACLs as we do not need them. Do not fail if a tool sets ACLs.
+        BlockPublicAcls: true
+        IgnorePublicAcls: true
+        # We want a public policy for GET requests (see below)
+        BlockPublicPolicy: false
+        RestrictPublicBuckets: false
+    UpdateReplacePolicy: Retain
+    DeletionPolicy: Retain
+  BootStrapBucketPolicy:
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      Bucket:
+        Ref: BootStrapBucket
+      PolicyDocument:
+        Statement:
+          - Action: s3:GetObject
+            Effect: Allow
+            Principal:
+              AWS: "*"
+            Resource:
+              Fn::Join:
+                - ""
+                - - Fn::GetAtt:
+                      - BootStrapBucket
+                      - Arn
+                  - /*
+        Version: "2012-10-17"

cluster.ts

@@ -11,70 +11,38 @@ import { ECRRepository } from "./ecr";
 import { KubernetesVersion } from "aws-cdk-lib/aws-eks";
 import { DeployImagePullSecret } from "./pullsecret";
 import { ArnPrincipal } from "aws-cdk-lib/aws-iam";
-import { StackProps } from "aws-cdk-lib";
-
-const accountID = process.env.CDK_DEFAULT_ACCOUNT!;
-const gitUrl = "https://github.com/aws-samples/eks-blueprints-workloads.git";
 
+const PARAMETER_SUBDOMAIN = "subdomain"
+const PARAMETER_HOSTEDZONEID = "hostedZoneId"
 /**
  * See docs/patterns/nginx.md for mode details on the setup.
  */
-export default class DevCluster extends cdk.Stack {
-  async eksCluster(scope: Construct, id: string, props: StackProps) {
-    const accountId = cdk.Stack.of(this).account
-    const region = cdk.Stack.of(this).region
-
-
-    const subdomainName = new cdk.CfnParameter(this, "subdomain", {
-      type: "String",
-      description: "The subdomain that can be used for ingress to the development environments\
-       e.g. garden.mycompany.com. \
-       Needs to be a hosted domain in Route53RecordTarget."}).valueAsString;
-
-    //unfortunately we can't use this lookup for env agnostic stacks (need to specify region)
-    // https://docs.aws.amazon.com/cdk/v2/guide/resources.html#resources_external
-    // const domainID = route53.HostedZone.fromLookup(this, id, {domainName: subdomainName})
-
-    const hostedZoneId = new cdk.CfnParameter(this, "hostedZoneId", {
-      type: "AWS::Route53::HostedZone::Id",
-      description: "The ID of the Route53 hosted zone with the domain that can be used for ingress\
-       to the development environments"}).valueAsString;
-
-    const iamUsers = new cdk.CfnParameter(this, "iamUsers", {
-      type: "CommaDelimitedList",
-      default: "",
-      description: "Comma delimited list of IAM users principal ARNs that should get access to the dev cluster\
-       e.g. \"arn:aws:iam::123456789012:user/JohnDoe,arn:aws:iam::123456789012:user/Alice\""}).valueAsList;
-
-    const iamRole = new cdk.CfnParameter(this, "iamRole", {
-      type: "String",
-      default: "arn:aws:sts::049586690729:assumed-role/AWSReservedSSO_AdministratorAccess_b3c1cae6dc09120a",
-      description: "A role that should get access to the dev cluster e.g.\
-       \"arn:aws:iam::123456789023:role/AWSReservedSSO_PlatformEngineers_4ed12acae0543\""}).valueAsString;
-
-    // use context variables for now because we cannot use CfnParameters
-    // for these values since we are manipulating the values in the code
-    const iamUsersArnsCtx = this.node.tryGetContext('iamUsers').split(",").map((item: string) => new ArnPrincipal(item))
-    const iamRoleArnCtx = this.node.tryGetContext('iamRole')
-    const subdomainNameCtx = this.node.tryGetContext('subdomainName')
-    const hostedZoneIdCtx = this.node.tryGetContext('hostedZoneId')
+export class DevClusterConstruct {
+  async eksCluster(scope: Construct, id: string) {
+    // TODO
+    const iamRoleArnCtx = "arn:aws:iam::123456789023:role/AWSReservedSSO_PlatformEngineers_4ed12acae0543"
+    const iamUsersArnsCtx = [new ArnPrincipal(iamRoleArnCtx)]
     const teams: Array<blueprints.Team> = [new TeamPlatform({userRoleArn: iamRoleArnCtx, users: iamUsersArnsCtx})];
 
     blueprints.HelmAddOn.validateHelmVersions = false;
+
+    const accountId = cdk.Fn.sub("${AWS::AccountId}")
+    const region = cdk.Fn.sub("${AWS::Region}")
+
     const cluster = await blueprints.EksBlueprint.builder()
       .account(accountId)
       .region(region)
       .version(KubernetesVersion.V1_24)
       .teams(...teams)
       .resourceProvider(
         GlobalResources.HostedZone,
-        new ImportHostedZoneProvider(hostedZoneIdCtx, subdomainNameCtx)
+        new ImportHostedZoneProvider(cdk.Fn.ref(PARAMETER_HOSTEDZONEID), cdk.Fn.ref(PARAMETER_SUBDOMAIN))
       )
       .resourceProvider(
         GlobalResources.Certificate,
         new blueprints.CreateCertificateProvider(
           "wildcard-cert",
-          `*${subdomainNameCtx}`,
+          cdk.Fn.join(".", ["*", cdk.Fn.ref(PARAMETER_SUBDOMAIN)]),
           GlobalResources.HostedZone
         )
       )
@@ -90,19 +58,41 @@ export default class DevCluster extends cdk.Stack {
           version: "0.15.2",
           internetFacing: true,
           backendProtocol: "tcp",
-          externalDnsHostname: subdomainNameCtx,
+          externalDnsHostname: cdk.Fn.ref(PARAMETER_SUBDOMAIN),
           crossZoneEnabled: false,
           certificateResourceName: GlobalResources.Certificate,
         }),
         new blueprints.SecretsStoreAddOn({ rotationPollInterval: "120s" }),
         new blueprints.ClusterAutoScalerAddOn(),
-        new DeployImagePullSecret({accountId: accountId, region: region}),
+        new DeployImagePullSecret({ accountId, region }),
         new blueprints.NestedStackAddOn({
           builder: ECRRepository.builder(),
           id: "ecr-nested-stack"
         })
       )
-      .buildAsync(scope, `${id}`);
+      .buildAsync(scope, id);
+
+      // add parameters
+      // TODO: is there a better approach to add parameters to the EKS blueprint CFN stack?
+
+      new cdk.CfnParameter(cluster, PARAMETER_SUBDOMAIN, {
+        type: "String",
+        description: `
+          The subdomain that can be used for ingress to the development environments, e.g. garden.mycompany.com
+          Needs to be a hosted domain in Route53RecordTarget.
+        `
+      });
+
+    // unfortunately we can't use this lookup for env agnostic stacks (need to specify region)
+    // https://docs.aws.amazon.com/cdk/v2/guide/resources.html#resources_external
+    // const domainID = route53.HostedZone.fromLookup(this, id, {domainName: subdomainName})
+    new cdk.CfnParameter(cluster, PARAMETER_HOSTEDZONEID, {
+      type: "AWS::Route53::HostedZone::Id",
+      description: `
+        The ID of the Route53 hosted zone with the domain that can be used for ingress
+        to the development environments
+      `
+    });
 
     blueprints.HelmAddOn.validateHelmVersions = false;
     cluster

ecr.ts

@@ -19,10 +19,10 @@ export class ECRRepository extends cdk.NestedStack {
     this.ecrRepoNames = ["api", "vote", "worker", "result"]
     for (var repo of this.ecrRepoNames) {
       this.ecrRepos.push(new ecr.Repository(this, repo, {
-          repositoryName: `garden-demo/${repo}`,
+          repositoryName: `garden-dev-cluster/${repo}`,
           encryption: ecr.RepositoryEncryption.KMS,
           imageScanOnPush: true,
       }));
     }
   }
-}
+}

main.ts

@@ -1,11 +1,27 @@
 import * as cdk from "aws-cdk-lib";
 import { logger } from "@aws-quickstart/eks-blueprints/dist/utils";
 import { HelmAddOn } from "@aws-quickstart/eks-blueprints";
-import DevCluster from "./cluster";
+import { DevClusterConstruct } from "./cluster";
+
+const releaseVersion = process.env["CDK_RELEASE_VERSION"]
+
+let defaultStackSynthesizer: cdk.IReusableStackSynthesizer | undefined
+
+// if this is being synthesized during the release process, we want to use our public S3 buckets.
+// otherwise (by default) you can just use the CDK deploy command to test the stack, e.g. for testing
+if (releaseVersion) {
+  defaultStackSynthesizer = new cdk.CliCredentialsStackSynthesizer({
+    // see also boostrap/README.md for more information about the nature of these S3 buckets
+    fileAssetsBucketName: 'garden-cfn-public-${AWS::Region}',
+    bucketPrefix: `dev-cluster/${releaseVersion}`,
+  })
+}
+
+const app = new cdk.App({ defaultStackSynthesizer });
 
-const app = new cdk.App();
 HelmAddOn.validateHelmVersions = false;
 
-new DevCluster().eksCluster(app, `dev-cluster`, {crossRegionReferences: true}).catch(() => {
+// main stack based on AWS EKS blueprints
+new DevClusterConstruct().eksCluster(app, `garden-dev-cluster`).catch(() => {
   logger.info("Error setting up dev cluster");
 });