From 9efb8fc392294272e6528fe08cc15a42d489af9b Mon Sep 17 00:00:00 2001 From: Vara Bonthu Date: Tue, 16 Jul 2024 15:56:04 -0700 Subject: [PATCH] Deprecating the blueprints --- ai-ml/kubeflow/README.md | 1 - ai-ml/ray/terraform/README.md | 56 --- ai-ml/ray/terraform/cleanup.sh | 28 -- .../examples/pytorch/job/pytorch_submit.py | 20 -- ai-ml/ray/terraform/examples/pytorch/main.tf | 122 ------- .../ray/terraform/examples/pytorch/outputs.tf | 0 .../terraform/examples/pytorch/variables.tf | 5 - .../terraform/examples/pytorch/versions.tf | 18 - .../examples/xgboost/job/xgboost_submit.py | 21 -- ai-ml/ray/terraform/examples/xgboost/main.tf | 115 ------ .../ray/terraform/examples/xgboost/outputs.tf | 0 .../terraform/examples/xgboost/variables.tf | 5 - .../terraform/examples/xgboost/versions.tf | 18 - ai-ml/ray/terraform/install.sh | 14 - ai-ml/ray/terraform/main.tf | 321 ----------------- .../terraform/modules/ray-cluster/README.md | 38 -- .../ray/terraform/modules/ray-cluster/main.tf | 95 ----- .../terraform/modules/ray-cluster/outputs.tf | 0 .../modules/ray-cluster/variables.tf | 26 -- .../terraform/modules/ray-cluster/versions.tf | 14 - ai-ml/ray/terraform/outputs.tf | 0 ai-ml/ray/terraform/variables.tf | 17 - ai-ml/ray/terraform/versions.tf | 22 -- ai-ml/ray/terraform/vpc.tf | 45 --- analytics/cdk/emr-eks/.gitignore | 8 - analytics/cdk/emr-eks/.npmignore | 6 - analytics/cdk/emr-eks/README.md | 3 - analytics/cdk/emr-eks/bin/emr-eks.ts | 43 --- analytics/cdk/emr-eks/cdk.json | 42 --- analytics/cdk/emr-eks/jest.config.js | 8 - .../emr-eks/lib/emr-eks-blueprint-stack.ts | 99 ----- analytics/cdk/emr-eks/lib/vpc.ts | 20 -- analytics/cdk/emr-eks/package.json | 31 -- .../emr-eks/test/emr-eks-blueprint.test.ts | 62 ---- analytics/cdk/emr-eks/test/emr-eks.test.ts | 70 ---- analytics/cdk/emr-eks/tsconfig.json | 38 -- analytics/cdk/stream-emr-on-eks/.gitignore | 41 --- .../cdk/stream-emr-on-eks/CODE_OF_CONDUCT.md | 4 - .../cdk/stream-emr-on-eks/CONTRIBUTING.md | 59 --- analytics/cdk/stream-emr-on-eks/Dockerfile | 5 - analytics/cdk/stream-emr-on-eks/LICENSE | 14 - analytics/cdk/stream-emr-on-eks/README.md | 340 ------------------ analytics/cdk/stream-emr-on-eks/app.py | 35 -- analytics/cdk/stream-emr-on-eks/cdk.json | 9 - .../deployment/app_code/delete_all.sh | 23 -- .../app_code/job/driver_template.yaml | 6 - .../deployment/app_code/job/emr-mount-efs.sh | 55 --- .../app_code/job/executor_template.yaml | 5 - .../deployment/app_code/job/msk_consumer.py | 62 ---- .../app_code/job/pyspark-kinesis.ipynb | 205 ----------- .../app_code/job/pyspark-kinesis.py | 63 ---- .../app_code/job/pyspark-kinesis_submit.sh | 27 -- .../app_code/job/qubole-kinesis.ipynb | 174 --------- .../deployment/app_code/job/qubole-kinesis.py | 79 ---- .../app_code/job/qubole-kinesis_submit.sh | 27 -- .../deployment/app_code/job/wordcount.py | 6 - .../deployment/app_code/post-deployment.sh | 60 ---- .../deployment/build-s3-dist.sh | 241 ------------- .../deployment/cdk-solution-helper/README.md | 152 -------- .../deployment/cdk-solution-helper/index.js | 120 ------- .../cdk-solution-helper/package.json | 11 - analytics/cdk/stream-emr-on-eks/package.json | 5 - .../cdk/stream-emr-on-eks/requirements.txt | 2 - analytics/cdk/stream-emr-on-eks/setup.py | 52 --- .../source/app_resources/00-deploy-to-aws.png | Bin 908 -> 0 bytes .../source/app_resources/alb-iam-role.yaml | 166 --------- .../source/app_resources/alb-values.yaml | 6 - .../app_resources/autoscaler-iam-role.yaml | 11 - .../app_resources/autoscaler-values.yaml | 17 - .../source/app_resources/emr-iam-role.yaml | 43 --- .../source/app_resources/emr-rbac.yaml | 42 --- .../app_resources/native-spark-iam-role.yaml | 25 -- .../app_resources/native-spark-rbac.yaml | 13 - .../app_resources/spark-operator-values.yaml | 11 - .../source/lib/cdk_infra/eks_base_app.py | 71 ---- .../source/lib/cdk_infra/eks_cluster.py | 83 ----- .../lib/cdk_infra/eks_service_account.py | 46 --- .../source/lib/cdk_infra/iam_roles.py | 115 ------ .../source/lib/cdk_infra/network_sg.py | 46 --- .../source/lib/cdk_infra/s3_app_code.py | 31 -- .../source/lib/cdk_infra/spark_permission.py | 207 ----------- .../source/lib/emr_on_ec2_stack.py | 157 -------- .../stream-emr-on-eks/source/lib/msk_stack.py | 60 ---- .../source/lib/spark_on_eks_stack.py | 52 --- .../source/lib/util/manifest_reader.py | 91 ----- .../cdk/stream-emr-on-eks/source/package.json | 5 - analytics/terraform/emr-eks-ack/README.md | 64 ---- analytics/terraform/emr-eks-ack/addons.tf | 72 ---- analytics/terraform/emr-eks-ack/cleanup.sh | 53 --- analytics/terraform/emr-eks-ack/emr-ack.tf | 8 - analytics/terraform/emr-eks-ack/emr-eks.tf | 19 - .../examples/emr-virtualcluster.yaml | 13 - .../examples/sample-pyspark-job.sh | 16 - .../examples/sample-pyspark-job.yaml | 23 -- .../helm-values/aws-for-fluentbit-values.yaml | 80 ----- analytics/terraform/emr-eks-ack/install.sh | 39 -- analytics/terraform/emr-eks-ack/main.tf | 148 -------- .../emr-eks-ack/modules/emr-ack/README.md | 48 --- .../emr-eks-ack/modules/emr-ack/main.tf | 165 --------- .../emr-eks-ack/modules/emr-ack/outputs.tf | 0 .../emr-eks-ack/modules/emr-ack/variables.tf | 31 -- .../emr-eks-ack/modules/emr-ack/versions.tf | 14 - analytics/terraform/emr-eks-ack/outputs.tf | 32 -- analytics/terraform/emr-eks-ack/variables.tf | 42 --- analytics/terraform/emr-eks-ack/versions.tf | 25 -- analytics/terraform/emr-eks-ack/vpc.tf | 40 --- analytics/terraform/emr-eks-fargate/README.md | 56 --- analytics/terraform/emr-eks-fargate/addons.tf | 58 --- .../terraform/emr-eks-fargate/cleanup.sh | 52 --- .../terraform/emr-eks-fargate/emr-eks.tf | 19 - .../examples/basic-pyspark-job.sh | 53 --- .../terraform/emr-eks-fargate/install.sh | 35 -- analytics/terraform/emr-eks-fargate/main.tf | 82 ----- .../terraform/emr-eks-fargate/outputs.tf | 9 - .../terraform/emr-eks-fargate/variables.tf | 42 --- .../terraform/emr-eks-fargate/versions.tf | 25 -- analytics/terraform/emr-eks-fargate/vpc.tf | 38 -- distributed-databases/cassandra/README.md | 1 - distributed-databases/cockroachdb/README.md | 1 - distributed-databases/mongodb/README.md | 1 - website/docs/blueprints/ai-ml/jark.md | 147 ++++++++ website/docs/blueprints/ai-ml/ray.md | 317 ---------------- website/docs/blueprints/ai-ml/trainium.md | 4 +- .../amazon-emr-on-eks/emr-eks-ack.md | 153 -------- .../amazon-emr-on-eks/emr-eks-cdk.md | 158 -------- .../amazon-emr-on-eks/emr-eks-fargate.md | 129 ------- .../emr-eks-observability.md | 2 +- .../emr-eks-spark-operator.md | 2 +- .../amazon-emr-on-eks/emr-eks-studio.md | 202 ----------- 129 files changed, 151 insertions(+), 6978 deletions(-) delete mode 100755 ai-ml/kubeflow/README.md delete mode 100644 ai-ml/ray/terraform/README.md delete mode 100755 ai-ml/ray/terraform/cleanup.sh delete mode 100644 ai-ml/ray/terraform/examples/pytorch/job/pytorch_submit.py delete mode 100644 ai-ml/ray/terraform/examples/pytorch/main.tf delete mode 100644 ai-ml/ray/terraform/examples/pytorch/outputs.tf delete mode 100644 ai-ml/ray/terraform/examples/pytorch/variables.tf delete mode 100644 ai-ml/ray/terraform/examples/pytorch/versions.tf delete mode 100644 ai-ml/ray/terraform/examples/xgboost/job/xgboost_submit.py delete mode 100644 ai-ml/ray/terraform/examples/xgboost/main.tf delete mode 100644 ai-ml/ray/terraform/examples/xgboost/outputs.tf delete mode 100644 ai-ml/ray/terraform/examples/xgboost/variables.tf delete mode 100644 ai-ml/ray/terraform/examples/xgboost/versions.tf delete mode 100755 ai-ml/ray/terraform/install.sh delete mode 100644 ai-ml/ray/terraform/main.tf delete mode 100644 ai-ml/ray/terraform/modules/ray-cluster/README.md delete mode 100644 ai-ml/ray/terraform/modules/ray-cluster/main.tf delete mode 100644 ai-ml/ray/terraform/modules/ray-cluster/outputs.tf delete mode 100644 ai-ml/ray/terraform/modules/ray-cluster/variables.tf delete mode 100644 ai-ml/ray/terraform/modules/ray-cluster/versions.tf delete mode 100644 ai-ml/ray/terraform/outputs.tf delete mode 100644 ai-ml/ray/terraform/variables.tf delete mode 100644 ai-ml/ray/terraform/versions.tf delete mode 100644 ai-ml/ray/terraform/vpc.tf delete mode 100644 analytics/cdk/emr-eks/.gitignore delete mode 100644 analytics/cdk/emr-eks/.npmignore delete mode 100644 analytics/cdk/emr-eks/README.md delete mode 100644 analytics/cdk/emr-eks/bin/emr-eks.ts delete mode 100644 analytics/cdk/emr-eks/cdk.json delete mode 100644 analytics/cdk/emr-eks/jest.config.js delete mode 100644 analytics/cdk/emr-eks/lib/emr-eks-blueprint-stack.ts delete mode 100644 analytics/cdk/emr-eks/lib/vpc.ts delete mode 100644 analytics/cdk/emr-eks/package.json delete mode 100644 analytics/cdk/emr-eks/test/emr-eks-blueprint.test.ts delete mode 100644 analytics/cdk/emr-eks/test/emr-eks.test.ts delete mode 100644 analytics/cdk/emr-eks/tsconfig.json delete mode 100644 analytics/cdk/stream-emr-on-eks/.gitignore delete mode 100644 analytics/cdk/stream-emr-on-eks/CODE_OF_CONDUCT.md delete mode 100644 analytics/cdk/stream-emr-on-eks/CONTRIBUTING.md delete mode 100644 analytics/cdk/stream-emr-on-eks/Dockerfile delete mode 100644 analytics/cdk/stream-emr-on-eks/LICENSE delete mode 100644 analytics/cdk/stream-emr-on-eks/README.md delete mode 100644 analytics/cdk/stream-emr-on-eks/app.py delete mode 100644 analytics/cdk/stream-emr-on-eks/cdk.json delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/app_code/delete_all.sh delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/driver_template.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/emr-mount-efs.sh delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/executor_template.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/msk_consumer.py delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.ipynb delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.py delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis_submit.sh delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.ipynb delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.py delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis_submit.sh delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/app_code/job/wordcount.py delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/app_code/post-deployment.sh delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/build-s3-dist.sh delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/README.md delete mode 100755 analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/index.js delete mode 100644 analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/package.json delete mode 100644 analytics/cdk/stream-emr-on-eks/package.json delete mode 100644 analytics/cdk/stream-emr-on-eks/requirements.txt delete mode 100644 analytics/cdk/stream-emr-on-eks/setup.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/00-deploy-to-aws.png delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/alb-iam-role.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/alb-values.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-iam-role.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-values.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/emr-iam-role.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/emr-rbac.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-iam-role.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-rbac.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/app_resources/spark-operator-values.yaml delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_base_app.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_cluster.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_service_account.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/iam_roles.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/network_sg.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/s3_app_code.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/spark_permission.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/emr_on_ec2_stack.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/msk_stack.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/spark_on_eks_stack.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/lib/util/manifest_reader.py delete mode 100644 analytics/cdk/stream-emr-on-eks/source/package.json delete mode 100644 analytics/terraform/emr-eks-ack/README.md delete mode 100644 analytics/terraform/emr-eks-ack/addons.tf delete mode 100755 analytics/terraform/emr-eks-ack/cleanup.sh delete mode 100644 analytics/terraform/emr-eks-ack/emr-ack.tf delete mode 100644 analytics/terraform/emr-eks-ack/emr-eks.tf delete mode 100644 analytics/terraform/emr-eks-ack/examples/emr-virtualcluster.yaml delete mode 100644 analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.sh delete mode 100644 analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.yaml delete mode 100755 analytics/terraform/emr-eks-ack/helm-values/aws-for-fluentbit-values.yaml delete mode 100755 analytics/terraform/emr-eks-ack/install.sh delete mode 100644 analytics/terraform/emr-eks-ack/main.tf delete mode 100644 analytics/terraform/emr-eks-ack/modules/emr-ack/README.md delete mode 100644 analytics/terraform/emr-eks-ack/modules/emr-ack/main.tf delete mode 100644 analytics/terraform/emr-eks-ack/modules/emr-ack/outputs.tf delete mode 100644 analytics/terraform/emr-eks-ack/modules/emr-ack/variables.tf delete mode 100644 analytics/terraform/emr-eks-ack/modules/emr-ack/versions.tf delete mode 100644 analytics/terraform/emr-eks-ack/outputs.tf delete mode 100644 analytics/terraform/emr-eks-ack/variables.tf delete mode 100644 analytics/terraform/emr-eks-ack/versions.tf delete mode 100644 analytics/terraform/emr-eks-ack/vpc.tf delete mode 100644 analytics/terraform/emr-eks-fargate/README.md delete mode 100644 analytics/terraform/emr-eks-fargate/addons.tf delete mode 100755 analytics/terraform/emr-eks-fargate/cleanup.sh delete mode 100644 analytics/terraform/emr-eks-fargate/emr-eks.tf delete mode 100755 analytics/terraform/emr-eks-fargate/examples/basic-pyspark-job.sh delete mode 100755 analytics/terraform/emr-eks-fargate/install.sh delete mode 100644 analytics/terraform/emr-eks-fargate/main.tf delete mode 100644 analytics/terraform/emr-eks-fargate/outputs.tf delete mode 100644 analytics/terraform/emr-eks-fargate/variables.tf delete mode 100644 analytics/terraform/emr-eks-fargate/versions.tf delete mode 100644 analytics/terraform/emr-eks-fargate/vpc.tf delete mode 100755 distributed-databases/cassandra/README.md delete mode 100755 distributed-databases/cockroachdb/README.md delete mode 100755 distributed-databases/mongodb/README.md create mode 100644 website/docs/blueprints/ai-ml/jark.md delete mode 100644 website/docs/blueprints/ai-ml/ray.md delete mode 100644 website/docs/blueprints/amazon-emr-on-eks/emr-eks-ack.md delete mode 100644 website/docs/blueprints/amazon-emr-on-eks/emr-eks-cdk.md delete mode 100644 website/docs/blueprints/amazon-emr-on-eks/emr-eks-fargate.md delete mode 100644 website/docs/blueprints/amazon-emr-on-eks/emr-eks-studio.md diff --git a/ai-ml/kubeflow/README.md b/ai-ml/kubeflow/README.md deleted file mode 100755 index 4e01e06fd..000000000 --- a/ai-ml/kubeflow/README.md +++ /dev/null @@ -1 +0,0 @@ -# KubeFlow on EKS (Coming Soon) diff --git a/ai-ml/ray/terraform/README.md b/ai-ml/ray/terraform/README.md deleted file mode 100644 index 68d80a914..000000000 --- a/ai-ml/ray/terraform/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Ray with Amazon EKS -Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ai-ml/ray) to deploy this pattern and run sample tests. - ---- - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [aws](#requirement\_aws) | >= 3.72 | -| [helm](#requirement\_helm) | >= 2.4.1 | -| [kubectl](#requirement\_kubectl) | >= 1.14 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.72 | -| [aws.ecr\_public\_region](#provider\_aws.ecr\_public\_region) | >= 3.72 | -| [helm](#provider\_helm) | >= 2.4.1 | -| [kubectl](#provider\_kubectl) | >= 1.14 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | -| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints-addons | 08650fd2b4bc894bde7b51313a8dc9598d82e925 | -| [karpenter](#module\_karpenter) | terraform-aws-modules/eks/aws//modules/karpenter | ~> 19.15 | -| [karpenter\_policy](#module\_karpenter\_policy) | terraform-aws-modules/iam/aws//modules/iam-policy | ~> 5.20 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | - -## Resources - -| Name | Type | -|------|------| -| [helm_release.kuberay_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubectl_manifest.eni_config](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.25"` | no | -| [name](#input\_name) | Name of the VPC, EKS Cluster and Ray cluster | `string` | `"ray-cluster"` | no | -| [region](#input\_region) | Region | `string` | `"us-west-2"` | no | - -## Outputs - -No outputs. - diff --git a/ai-ml/ray/terraform/cleanup.sh b/ai-ml/ray/terraform/cleanup.sh deleted file mode 100755 index f9c7cdc72..000000000 --- a/ai-ml/ray/terraform/cleanup.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail - -targets=( - "module.eks_blueprints_addons" - "module.eks" - "module.vpc" -) - -for target in "${targets[@]}" -do - destroy_output=$(terraform destroy -target="$target" -auto-approve 2>&1) - if [[ $? -eq 0 && $destroy_output == *"Destroy complete!"* ]]; then - echo "SUCCESS: Terraform destroy of $target completed successfully" - else - echo "FAILED: Terraform destroy of $target failed" - exit 1 - fi -done - -destroy_output=$(terraform destroy -auto-approve 2>&1) -if [[ $? -eq 0 && $destroy_output == *"Destroy complete!"* ]]; then - echo "SUCCESS: Terraform destroy of all targets completed successfully" -else - echo "FAILED: Terraform destroy of all targets failed" - exit 1 -fi diff --git a/ai-ml/ray/terraform/examples/pytorch/job/pytorch_submit.py b/ai-ml/ray/terraform/examples/pytorch/job/pytorch_submit.py deleted file mode 100644 index 90789c635..000000000 --- a/ai-ml/ray/terraform/examples/pytorch/job/pytorch_submit.py +++ /dev/null @@ -1,20 +0,0 @@ -from ray.job_submission import JobSubmissionClient - -address = "http://127.0.0.1:8266" - -client = JobSubmissionClient(address) - -kick_off_pytorch_benchmark = ( - # Clone ray. If ray is already present, don't clone again. - "git clone https://github.com/ray-project/ray || true;" - # Run the benchmark. - " python ray/release/air_tests/air_benchmarks/workloads/tune_torch_benchmark.py" -) - - -submission_id = client.submit_job( - entrypoint=kick_off_pytorch_benchmark, -) - -print("Use the following command to follow this Job's logs:") -print(f"ray job logs '{submission_id}' --follow --address {address}") diff --git a/ai-ml/ray/terraform/examples/pytorch/main.tf b/ai-ml/ray/terraform/examples/pytorch/main.tf deleted file mode 100644 index f11f7ebe9..000000000 --- a/ai-ml/ray/terraform/examples/pytorch/main.tf +++ /dev/null @@ -1,122 +0,0 @@ -provider "aws" { - region = local.region -} - -provider "kubernetes" { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", local.eks_cluster] - } -} - -provider "helm" { - kubernetes { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", local.eks_cluster] - } - } -} - -data "aws_eks_cluster" "this" { - name = local.eks_cluster -} - -locals { - region = var.region - name = "pytorch" - eks_cluster = "ray-cluster" -} - -module "pytorch_cluster" { - source = "../../modules/ray-cluster" - - namespace = local.name - ray_cluster_name = local.name - eks_cluster_name = local.eks_cluster - - helm_values = [ - yamlencode({ - image = { - repository = "rayproject/ray-ml" - # This is a different version than the xgboost version - tag = "2.3.0" - pullPolicy = "IfNotPresent" - } - head = { - enableInTreeAutoscaling = "True" - resources = { - limits = { - cpu = "4" - memory = "24G" - } - requests = { - cpu = "4" - memory = "12G" - } - } - tolerations = [ - { - key = local.name - effect = "NoSchedule" - operator = "Exists" - } - ] - containerEnv = [ - { - name = "RAY_LOG_TO_STDERR" - value = "1" - }, - { - # workaround for protobuf protoc >= 3.19.0 issue - name = "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION" - value = "python" - } - ] - } - worker = { - resources = { - limits = { - cpu = "8" - memory = "24G" - } - requests = { - cpu = "4" - memory = "12G" - } - } - tolerations = [ - { - key = local.name - effect = "NoSchedule" - operator = "Exists" - } - ] - replicas = "0" - minReplicas = "0" - maxReplicas = "30" - containerEnv = [ - { - name = "RAY_LOG_TO_STDERR" - value = "1" - }, - { - # workaround for protobuf protoc >= 3.19.0 issue - name = "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION" - value = "python" - } - ] - } - }) - ] -} diff --git a/ai-ml/ray/terraform/examples/pytorch/outputs.tf b/ai-ml/ray/terraform/examples/pytorch/outputs.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/ai-ml/ray/terraform/examples/pytorch/variables.tf b/ai-ml/ray/terraform/examples/pytorch/variables.tf deleted file mode 100644 index 9225aa7b3..000000000 --- a/ai-ml/ray/terraform/examples/pytorch/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "Region" - type = string - default = "us-west-2" -} diff --git a/ai-ml/ray/terraform/examples/pytorch/versions.tf b/ai-ml/ray/terraform/examples/pytorch/versions.tf deleted file mode 100644 index 9ac174272..000000000 --- a/ai-ml/ray/terraform/examples/pytorch/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.72" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4.1" - } - } -} diff --git a/ai-ml/ray/terraform/examples/xgboost/job/xgboost_submit.py b/ai-ml/ray/terraform/examples/xgboost/job/xgboost_submit.py deleted file mode 100644 index 97807348b..000000000 --- a/ai-ml/ray/terraform/examples/xgboost/job/xgboost_submit.py +++ /dev/null @@ -1,21 +0,0 @@ -from ray.job_submission import JobSubmissionClient - -address = "http://127.0.0.1:8265" - -client = JobSubmissionClient(address) - -kick_off_xgboost_benchmark = ( - # Clone ray. If ray is already present, don't clone again. - "git clone https://github.com/ray-project/ray || true;" - # Run the benchmark. - " python ray/release/air_tests/air_benchmarks/workloads/xgboost_benchmark.py" - " --size 100G --disable-check" -) - - -submission_id = client.submit_job( - entrypoint=kick_off_xgboost_benchmark, -) - -print("Use the following command to follow this Job's logs:") -print(f"ray job logs '{submission_id}' --follow --address {address}") diff --git a/ai-ml/ray/terraform/examples/xgboost/main.tf b/ai-ml/ray/terraform/examples/xgboost/main.tf deleted file mode 100644 index 1cebaea79..000000000 --- a/ai-ml/ray/terraform/examples/xgboost/main.tf +++ /dev/null @@ -1,115 +0,0 @@ -provider "aws" { - region = local.region -} - -provider "kubernetes" { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.example.certificate_authority[0].data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", local.eks_cluster] - } -} - -provider "helm" { - kubernetes { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", local.eks_cluster] - } - } -} - -data "aws_eks_cluster" "this" { - name = local.eks_cluster -} - -locals { - region = var.region - name = "xgboost" - eks_cluster = "ray-cluster" -} - -module "xgboost_cluster" { - source = "../../modules/ray-cluster" - - namespace = local.name - ray_cluster_name = local.name - eks_cluster_name = local.eks_cluster - - helm_values = [ - yamlencode({ - image = { - repository = "rayproject/ray-ml" - tag = "2.0.0" - pullPolicy = "IfNotPresent" - } - head = { - enableInTreeAutoscaling = "True" - resources = { - limits = { - cpu = "14" - memory = "54Gi" - ephemeral-storage = "700Gi" - } - requests = { - cpu = "14" - memory = "54Gi" - ephemeral-storage = "700Gi" - } - } - tolerations = [ - { - key = local.name - effect = "NoSchedule" - operator = "Exists" - } - ] - containerEnv = [ - { - name = "RAY_LOG_TO_STDERR" - value = "1" - } - ] - } - worker = { - resources = { - limits = { - cpu = "14" - memory = "54Gi" - ephemeral-storage = "700Gi" - } - requests = { - cpu = "14" - memory = "54Gi" - ephemeral-storage = "700Gi" - } - } - tolerations = [ - { - key = local.name - effect = "NoSchedule" - operator = "Exists" - } - ] - replicas = "0" - minReplicas = "0" - maxReplicas = "9" - containerEnv = [ - { - name = "RAY_LOG_TO_STDERR" - value = "1" - } - ] - } - }) - ] -} diff --git a/ai-ml/ray/terraform/examples/xgboost/outputs.tf b/ai-ml/ray/terraform/examples/xgboost/outputs.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/ai-ml/ray/terraform/examples/xgboost/variables.tf b/ai-ml/ray/terraform/examples/xgboost/variables.tf deleted file mode 100644 index 9225aa7b3..000000000 --- a/ai-ml/ray/terraform/examples/xgboost/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "Region" - type = string - default = "us-west-2" -} diff --git a/ai-ml/ray/terraform/examples/xgboost/versions.tf b/ai-ml/ray/terraform/examples/xgboost/versions.tf deleted file mode 100644 index 9ac174272..000000000 --- a/ai-ml/ray/terraform/examples/xgboost/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.72" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4.1" - } - } -} diff --git a/ai-ml/ray/terraform/install.sh b/ai-ml/ray/terraform/install.sh deleted file mode 100755 index 4f3be27fa..000000000 --- a/ai-ml/ray/terraform/install.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -echo "Initializing ..." -terraform init || echo "\"terraform init\" failed" - - -echo "Applying ..." -apply_output=$(terraform apply -auto-approve 2>&1 | tee /dev/tty) -if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then - echo "SUCCESS: Terraform apply completed successfully" -else - echo "FAILED: Terraform apply failed" - exit 1 -fi diff --git a/ai-ml/ray/terraform/main.tf b/ai-ml/ray/terraform/main.tf deleted file mode 100644 index d6e9f5b49..000000000 --- a/ai-ml/ray/terraform/main.tf +++ /dev/null @@ -1,321 +0,0 @@ -#--------------------------------------------------------------- -# Providers -#--------------------------------------------------------------- - -provider "aws" { - region = local.region -} - -# Used for Karpenter Helm chart -provider "aws" { - region = "us-east-1" - alias = "ecr_public_region" -} - -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } - } -} - -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - -#--------------------------------------------------------------- -# Data Sources -#--------------------------------------------------------------- - -data "aws_availability_zones" "available" {} - -# Used for Karpenter Helm chart -data "aws_ecrpublic_authorization_token" "token" { - provider = aws.ecr_public_region -} - -#--------------------------------------------------------------- -# Locals -#--------------------------------------------------------------- - -locals { - name = var.name - region = var.region - - vpc_cidr = "10.0.0.0/16" - secondary_vpc_cidr = "100.64.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - - cluster_version = var.eks_cluster_version - - tags = { - Blueprint = local.name - GithubRepo = "github.com/awslabs/data-on-eks" - } -} - -#--------------------------------------------------------------- -# EKS Cluster -#--------------------------------------------------------------- - -#tfsec:ignore:aws-eks-enable-control-plane-logging -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.15" - - cluster_name = local.name - cluster_version = local.cluster_version - #WARNING: Avoid using this option (cluster_endpoint_public_access = true) in preprod or prod accounts. This feature is designed for sandbox accounts, simplifying cluster deployment and testing. - cluster_endpoint_public_access = true - - vpc_id = module.vpc.vpc_id - # We only want to assign the 10.0.* range subnets to the data plane - subnet_ids = slice(module.vpc.private_subnets, 0, 3) - control_plane_subnet_ids = module.vpc.intra_subnets - - # Update aws-auth configmap with Karpenter node role so they - # can join the cluster - manage_aws_auth_configmap = true - aws_auth_roles = [ - { - rolearn = module.karpenter.role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, - ] - - # EKS Addons - cluster_addons = { - coredns = { - most_recent = true - } - kube-proxy = { - most_recent = true - } - vpc-cni = { - # The VPC CNI addon should be deployed before compute to ensure - # the addon is configured before data plane compute resources are created - # See README for further details - before_compute = true - most_recent = true # To ensure access to the latest settings provided - preserve = true - configuration_values = jsonencode({ - env = { - # Reference https://aws.github.io/aws-eks-best-practices/reliability/docs/networkmanagement/#cni-custom-networking - AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG = "true" - ENI_CONFIG_LABEL_DEF = "topology.kubernetes.io/zone" - - # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html - ENABLE_PREFIX_DELEGATION = "true" - WARM_PREFIX_TARGET = "1" - } - }) - } - } - - # This MNG will be used to host infrastructure add-ons for - # logging, monitoring, ingress controllers, kuberay-operator, - # etc. - eks_managed_node_groups = { - infra = { - instance_types = ["m5.large"] - min_size = 3 - max_size = 3 - desired_size = 3 - } - } - - tags = merge(local.tags, { - "karpenter.sh/discovery" = local.name - }) -} - -#--------------------------------------------------------------- -# VPC-CNI Custom Networking ENIConfig -#--------------------------------------------------------------- - -resource "kubectl_manifest" "eni_config" { - for_each = zipmap(local.azs, slice(module.vpc.private_subnets, 3, 6)) - - yaml_body = yamlencode({ - apiVersion = "crd.k8s.amazonaws.com/v1alpha1" - kind = "ENIConfig" - metadata = { - name = each.key - } - spec = { - securityGroups = [ - module.eks.cluster_primary_security_group_id, - module.eks.node_security_group_id, - ] - subnet = each.value - } - }) -} - -#--------------------------------------------------------------- -# Karpenter Infrastructure -#--------------------------------------------------------------- - -module "karpenter" { - source = "terraform-aws-modules/eks/aws//modules/karpenter" - version = "~> 19.15" - - cluster_name = module.eks.cluster_name - irsa_oidc_provider_arn = module.eks.oidc_provider_arn - create_irsa = false # IRSA will be created by the kubernetes-addons module - iam_role_additional_policies = { - additional_policy = module.karpenter_policy.arn - } - - tags = local.tags -} - -# We have to augment default the karpenter node IAM policy with -# permissions we need for Ray Jobs to run until IRSA is added -# upstream in kuberay-operator. See issue -# https://github.com/ray-project/kuberay/issues/746 -module "karpenter_policy" { - source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "~> 5.20" - - name = "KarpenterS3ReadOnlyPolicy" - description = "IAM Policy to allow read from an S3 bucket for karpenter nodes" - - policy = jsonencode( - { - Version = "2012-10-17" - Statement = [ - { - Sid = "ListObjectsInBucket" - Effect = "Allow" - Action = ["s3:ListBucket"] - Resource = ["arn:aws:s3:::air-example-data-2"] - }, - { - Sid = "AllObjectActions" - Effect = "Allow" - Action = "s3:Get*" - Resource = ["arn:aws:s3:::air-example-data-2/*"] - } - ] - } - ) -} - -#--------------------------------------------------------------- -# Operational Add-Ons using EKS Blueprints -#--------------------------------------------------------------- - -module "eks_blueprints_addons" { - # Users should pin the version to the latest available release - # tflint-ignore: terraform_module_pinned_source - source = "github.com/aws-ia/terraform-aws-eks-blueprints-addons?ref=08650fd2b4bc894bde7b51313a8dc9598d82e925" - - cluster_name = module.eks.cluster_name - cluster_endpoint = module.eks.cluster_endpoint - cluster_version = module.eks.cluster_version - oidc_provider = module.eks.oidc_provider - oidc_provider_arn = module.eks.oidc_provider_arn - - enable_cloudwatch_metrics = true - - enable_aws_for_fluentbit = true - aws_for_fluentbit_helm_config = { - aws_for_fluent_bit_cw_log_group = "/${local.name}/worker-fluentbit-logs" - aws_for_fluentbit_cwlog_retention_in_days = 7 #days - values = [ - yamlencode({ - name = "kubernetes" - match = "kube.*" - kubeURL = "https://kubernetes.default.svc.cluster.local:443" - mergeLog = "On" - mergeLogKey = "log_processed" - keepLog = "On" - k8sLoggingParser = "On" - k8sLoggingExclude = "Off" - bufferSize = "0" - hostNetwork = "true" - dnsPolicy = "ClusterFirstWithHostNet" - filter = { - extraFilters = <<-EOT - Kube_Tag_Prefix application.var.log.containers. - Labels Off - Annotations Off - Use_Kubelet true - Kubelet_Port 10250 - Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token - EOT - } - cloudWatch = { - enabled = "true" - match = "*" - region = local.region - logGroupName = "/${local.name}/worker-fluentbit-logs" - logStreamPrefix = "fluentbit-" - autoCreateGroup = "false" - } - }) - ] - } - - enable_karpenter = true - karpenter_helm_config = { - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password - } - karpenter_node_iam_instance_profile = module.karpenter.instance_profile_name - karpenter_enable_spot_termination_handling = true - - tags = local.tags -} - -#--------------------------------------------------------------- -# KubeRay Operator using Helm Release -#--------------------------------------------------------------- - -resource "helm_release" "kuberay_operator" { - namespace = "kuberay-operator" - create_namespace = true - name = "kuberay-operator" - repository = "https://ray-project.github.io/kuberay-helm/" - chart = "kuberay-operator" - version = "0.4.0" - - depends_on = [ - module.eks - ] -} diff --git a/ai-ml/ray/terraform/modules/ray-cluster/README.md b/ai-ml/ray/terraform/modules/ray-cluster/README.md deleted file mode 100644 index 13d67b118..000000000 --- a/ai-ml/ray/terraform/modules/ray-cluster/README.md +++ /dev/null @@ -1,38 +0,0 @@ -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [helm](#requirement\_helm) | >= 2.4.1 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | >= 2.4.1 | -| [kubectl](#provider\_kubectl) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.ray_cluster](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/hashicorp/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/hashicorp/kubectl/latest/docs/resources/manifest) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_name](#input\_cluster\_name) | Name of the Ray Cluster | `string` | n/a | yes | -| [eks\_cluster\_name](#input\_eks\_cluster\_name) | Name of the EKS Cluster | `string` | n/a | yes | -| [namespace](#input\_namespace) | Namespace for the Ray Cluster | `string` | n/a | yes | -| [values](#input\_values) | Helm Values map | `map(any)` | `{}` | no | - -## Outputs - -No outputs. diff --git a/ai-ml/ray/terraform/modules/ray-cluster/main.tf b/ai-ml/ray/terraform/modules/ray-cluster/main.tf deleted file mode 100644 index 5fcd9686d..000000000 --- a/ai-ml/ray/terraform/modules/ray-cluster/main.tf +++ /dev/null @@ -1,95 +0,0 @@ -# Disclaimer!!!! -# These are organizational specific configurations for the -# karpenter Provisioner, NodeTemplate and RayCluster packaged -# as a module for convenience. These should be parameterized as -# you see fit for your use-case. - -#--------------------------------------------------------------- -# Karpenter Configuration -#--------------------------------------------------------------- - -resource "kubectl_manifest" "karpenter_provisioner" { - yaml_body = yamlencode({ - apiVersion = "karpenter.sh/v1alpha5" - kind = "Provisioner" - metadata = { - name = var.ray_cluster_name - } - spec = { - requirements = [ - { - key = "karpenter.sh/capacity-type" - operator = "In" - values = ["on-demand"] - } - ] - limits = { - resources = { - cpu = "1000" - } - } - providerRef = { - name = var.ray_cluster_name - } - ttlSecondsAfterEmpty = 30 - taints = [ - { - key = var.ray_cluster_name - effect = "NoSchedule" - } - ] - } - }) -} - -resource "kubectl_manifest" "karpenter_node_template" { - yaml_body = yamlencode({ - apiVersion = "karpenter.k8s.aws/v1alpha1" - kind = "AWSNodeTemplate" - metadata = { - name = var.ray_cluster_name - } - spec = { - subnetSelector = { - "karpenter.sh/discovery" = var.eks_cluster_name - } - securityGroupSelector = { - "karpenter.sh/discovery" = var.eks_cluster_name - } - tags = { - "ray-cluster/name" = var.ray_cluster_name - "karpenter.sh/discovery" = var.eks_cluster_name - } - blockDeviceMappings = [ - { - deviceName = "/dev/xvda" - ebs = { - volumeSize = "1000Gi" - volumeType = "gp3" - deleteOnTermination = true - } - } - ] - } - }) -} - -#--------------------------------------------------------------- -# Ray Cluster -#--------------------------------------------------------------- - -resource "helm_release" "ray_cluster" { - namespace = var.namespace - create_namespace = true - name = var.ray_cluster_name - repository = "https://ray-project.github.io/kuberay-helm/" - chart = "ray-cluster" - version = var.ray_cluster_version - - values = var.helm_values - - depends_on = [ - kubectl_manifest.karpenter_node_template, - kubectl_manifest.karpenter_provisioner - ] -} diff --git a/ai-ml/ray/terraform/modules/ray-cluster/outputs.tf b/ai-ml/ray/terraform/modules/ray-cluster/outputs.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/ai-ml/ray/terraform/modules/ray-cluster/variables.tf b/ai-ml/ray/terraform/modules/ray-cluster/variables.tf deleted file mode 100644 index ffb1d57cf..000000000 --- a/ai-ml/ray/terraform/modules/ray-cluster/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "ray_cluster_name" { - description = "Name of the Ray Cluster" - type = string -} - -variable "eks_cluster_name" { - description = "Name of the EKS Cluster" - type = string -} - -variable "namespace" { - description = "Namespace for the Ray Cluster" - type = string -} - -variable "ray_cluster_version" { - description = "Namespace for the Ray Cluster" - type = string - default = "0.4.0" -} - -variable "helm_values" { - description = "Helm values for Ray Cluster helm chart" - type = list(any) - default = [] -} diff --git a/ai-ml/ray/terraform/modules/ray-cluster/versions.tf b/ai-ml/ray/terraform/modules/ray-cluster/versions.tf deleted file mode 100644 index df8e97260..000000000 --- a/ai-ml/ray/terraform/modules/ray-cluster/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - - required_providers { - helm = { - source = "hashicorp/helm" - version = ">= 2.4.1" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14" - } - } -} diff --git a/ai-ml/ray/terraform/outputs.tf b/ai-ml/ray/terraform/outputs.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/ai-ml/ray/terraform/variables.tf b/ai-ml/ray/terraform/variables.tf deleted file mode 100644 index 4e52b46d3..000000000 --- a/ai-ml/ray/terraform/variables.tf +++ /dev/null @@ -1,17 +0,0 @@ -variable "region" { - description = "Region" - type = string - default = "us-west-2" -} - -variable "name" { - description = "Name of the VPC, EKS Cluster and Ray cluster" - default = "ray-cluster" - type = string -} - -variable "eks_cluster_version" { - description = "EKS Cluster version" - default = "1.25" - type = string -} diff --git a/ai-ml/ray/terraform/versions.tf b/ai-ml/ray/terraform/versions.tf deleted file mode 100644 index 41659b984..000000000 --- a/ai-ml/ray/terraform/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.72" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4.1" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14" - } - } -} diff --git a/ai-ml/ray/terraform/vpc.tf b/ai-ml/ray/terraform/vpc.tf deleted file mode 100644 index a39e745fb..000000000 --- a/ai-ml/ray/terraform/vpc.tf +++ /dev/null @@ -1,45 +0,0 @@ -#--------------------------------------------------------------- -# VPC -#--------------------------------------------------------------- - -# WARNING: This VPC module includes the creation of an Internet Gateway and NAT Gateway, which simplifies cluster deployment and testing, primarily intended for sandbox accounts. -# IMPORTANT: For preprod and prod use cases, it is crucial to consult with your security team and AWS architects to design a private infrastructure solution that aligns with your security requirements - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = local.name - cidr = local.vpc_cidr - - secondary_cidr_blocks = [local.secondary_vpc_cidr] - - azs = local.azs - private_subnets = concat( - [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)], - [for k, v in local.azs : cidrsubnet(local.secondary_vpc_cidr, 4, k)] - ) - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] - # Control Plane Subnets - intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)] - - enable_nat_gateway = true - single_nat_gateway = true - - enable_flow_log = true - create_flow_log_cloudwatch_iam_role = true - create_flow_log_cloudwatch_log_group = true - - public_subnet_tags = { - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/role/internal-elb" = 1 - "karpenter.sh/discovery" = local.name - } - - tags = merge(local.tags, { - "karpenter.sh/discovery" = local.name - }) -} diff --git a/analytics/cdk/emr-eks/.gitignore b/analytics/cdk/emr-eks/.gitignore deleted file mode 100644 index f1b302482..000000000 --- a/analytics/cdk/emr-eks/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.js -!jest.config.js -*.d.ts -node_modules -dist -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/analytics/cdk/emr-eks/.npmignore b/analytics/cdk/emr-eks/.npmignore deleted file mode 100644 index c1d6d45dc..000000000 --- a/analytics/cdk/emr-eks/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -*.ts -!*.d.ts - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/analytics/cdk/emr-eks/README.md b/analytics/cdk/emr-eks/README.md deleted file mode 100644 index 56e030ce4..000000000 --- a/analytics/cdk/emr-eks/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Deploy EMR on EKS with CDK Blueprints - -For detailed instruction please visit https://awslabs.github.io/data-on-eks/docs/blueprints/amazon-emr-on-eks/emr-eks-cdk diff --git a/analytics/cdk/emr-eks/bin/emr-eks.ts b/analytics/cdk/emr-eks/bin/emr-eks.ts deleted file mode 100644 index 17e03873d..000000000 --- a/analytics/cdk/emr-eks/bin/emr-eks.ts +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env node -import 'source-map-support/register'; -import * as cdk from 'aws-cdk-lib'; -import { EmrEksTeamProps } from '@aws-quickstart/eks-blueprints'; -import { ArnPrincipal, PolicyStatement } from 'aws-cdk-lib/aws-iam'; -import EmrEksStack, { EmrEksBlueprintProps } from '../lib/emr-eks-blueprint-stack'; -import VpcDefinintion from '../lib/vpc'; - -const app = new cdk.App(); - -const account = process.env.CDK_DEFAULT_ACCOUNT; -const region = process.env.CDK_DEFAULT_REGION; - -const executionRolePolicyStatement: PolicyStatement[] = [ - new PolicyStatement({ - actions:['logs:PutLogEvents','logs:CreateLogStream','logs:DescribeLogGroups','logs:DescribeLogStreams'], - resources:['arn:aws:logs:*:*:*'], - }), -]; - -const dataTeamA: EmrEksTeamProps = { - name: 'emr-data-team-a', - virtualClusterName: 'emr-data-team-a', - virtualClusterNamespace: 'batchjob', - createNamespace: true, - executionRoles: [ - { - executionRoleIamPolicyStatement: executionRolePolicyStatement, - executionRoleName: 'myBlueprintExecRole' - } - ] -}; - -const vpc = new VpcDefinintion(app, 'vpcStack', {env: {account, region}} ).vpc; - -const props: EmrEksBlueprintProps = { - env: { account, region }, - dataTeams: [dataTeamA], - clusterAdminRoleArn: new ArnPrincipal('arn:aws:iam::11111111111:role/'), - clusterVpc: vpc -}; - -new EmrEksStack().build(app, 'data-on-eks', props); diff --git a/analytics/cdk/emr-eks/cdk.json b/analytics/cdk/emr-eks/cdk.json deleted file mode 100644 index 91f644f63..000000000 --- a/analytics/cdk/emr-eks/cdk.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "app": "npx ts-node --prefer-ts-exts bin/emr-eks.ts", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "**/*.d.ts", - "**/*.js", - "tsconfig.json", - "package*.json", - "yarn.lock", - "node_modules", - "test" - ] - }, - "context": { - "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, - "@aws-cdk/core:stackRelativeExports": true, - "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, - "@aws-cdk/aws-lambda:recognizeVersionProps": true, - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, - "@aws-cdk/core:enablePartitionLiterals": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ] - } -} diff --git a/analytics/cdk/emr-eks/jest.config.js b/analytics/cdk/emr-eks/jest.config.js deleted file mode 100644 index 08263b895..000000000 --- a/analytics/cdk/emr-eks/jest.config.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - testEnvironment: 'node', - roots: ['/test'], - testMatch: ['**/*.test.ts'], - transform: { - '^.+\\.tsx?$': 'ts-jest' - } -}; diff --git a/analytics/cdk/emr-eks/lib/emr-eks-blueprint-stack.ts b/analytics/cdk/emr-eks/lib/emr-eks-blueprint-stack.ts deleted file mode 100644 index 74d708af4..000000000 --- a/analytics/cdk/emr-eks/lib/emr-eks-blueprint-stack.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { Construct } from 'constructs'; -import * as blueprints from '@aws-quickstart/eks-blueprints' -import { - DirectVpcProvider, - GenericClusterProvider, - GlobalResources, - PlatformTeam, - EmrEksTeamProps, - EmrEksTeam, - EmrEksAddOn -} from '@aws-quickstart/eks-blueprints'; -import { CapacityType, ClusterLoggingTypes, KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks'; -import { InstanceType, IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; -import { StackProps } from 'aws-cdk-lib'; -import { ArnPrincipal } from 'aws-cdk-lib/aws-iam'; - -export interface EmrEksBlueprintProps extends StackProps { - clusterVpc: IVpc, - clusterAdminRoleArn: ArnPrincipal - dataTeams: EmrEksTeamProps[], - eksClusterName?: string, - eksCluster?: GenericClusterProvider, - -} - - -export default class EmrEksStack { - - build(scope: Construct, id: string, props: EmrEksBlueprintProps) { - - const eksClusterLogging: ClusterLoggingTypes[] = [ - ClusterLoggingTypes.API, - ClusterLoggingTypes.AUTHENTICATOR, - ClusterLoggingTypes.SCHEDULER, - ClusterLoggingTypes.CONTROLLER_MANAGER, - ClusterLoggingTypes.AUDIT, - ]; - - const emrCluster: GenericClusterProvider = new blueprints.GenericClusterProvider({ - clusterName: props.eksClusterName ? props.eksClusterName : 'eksBlueprintCluster', - version: KubernetesVersion.V1_23, - managedNodeGroups: [ - { - id: "core-node-grp", - nodeGroupCapacityType: CapacityType.ON_DEMAND, - amiType: NodegroupAmiType.AL2_X86_64, - instanceTypes: [new InstanceType('m5.xlarge')], - diskSize: 50 - }, - { - id: "spark-node-grp", - instanceTypes: [new InstanceType('r5d.large')], - nodeGroupCapacityType: CapacityType.ON_DEMAND, - amiType: NodegroupAmiType.AL2_X86_64, - diskSize: 50, - labels: { - app: 'spark' - }, - nodeGroupSubnets: {subnetType: SubnetType.PRIVATE_WITH_EGRESS, availabilityZones: [props.clusterVpc.availabilityZones[0]]} - - } - ], - clusterLogging: eksClusterLogging - }); - - const clusterAdminTeam = new PlatformTeam({ - name: "adminteam", - userRoleArn: props.clusterAdminRoleArn.arn - }); - - let emrEksBlueprint = blueprints.EksBlueprint.builder(); - - if (props.clusterVpc) { - emrEksBlueprint.resourceProvider(GlobalResources.Vpc, new DirectVpcProvider(props.clusterVpc)); - } - - let emrTeams: EmrEksTeam [] = [...props.dataTeams.map(team => new EmrEksTeam(team))]; - - emrEksBlueprint = props.eksCluster ? - emrEksBlueprint.clusterProvider(props.eksCluster) : - emrEksBlueprint.clusterProvider(emrCluster); - - return emrEksBlueprint.addOns( - new blueprints.VpcCniAddOn(), - new blueprints.CoreDnsAddOn(), - new blueprints.MetricsServerAddOn, - new blueprints.ClusterAutoScalerAddOn, - new blueprints.CertManagerAddOn, - new blueprints.AwsLoadBalancerControllerAddOn, - new blueprints.EbsCsiDriverAddOn, - new blueprints.KubeProxyAddOn, - new EmrEksAddOn) - .teams( - clusterAdminTeam, - ...emrTeams) - .build(scope, `${id}-emr-eks-blueprint`); - - } -} diff --git a/analytics/cdk/emr-eks/lib/vpc.ts b/analytics/cdk/emr-eks/lib/vpc.ts deleted file mode 100644 index c9b3d202a..000000000 --- a/analytics/cdk/emr-eks/lib/vpc.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { IpAddresses, Vpc } from 'aws-cdk-lib/aws-ec2'; -import { App, Stack, StackProps } from 'aws-cdk-lib'; - - -export default class VpcDefinintion extends Stack { - - public readonly vpc: Vpc; - - constructor(scope: App, id: string, props: StackProps) { - super(scope, id, props); - - this.vpc = new Vpc(this, 'eksVpc', { - ipAddresses: IpAddresses.cidr('10.0.0.0/16'), - natGateways: 3, - maxAzs: 3 - }); - - } - -} diff --git a/analytics/cdk/emr-eks/package.json b/analytics/cdk/emr-eks/package.json deleted file mode 100644 index 8ec29912e..000000000 --- a/analytics/cdk/emr-eks/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "emr-eks", - "version": "0.1.0", - "bin": { - "emr-eks": "dist/bin/emr-eks.js" - }, - "scripts": { - "build": "rm -rf dist && tsc", - "watch": "tsc -w", - "test": "jest", - "cdk": "cdk", - "lint": "npx eslint . --ext .js,.jsx,.ts,.tsx" - }, - "devDependencies": { - "@types/jest": "^27.5.2", - "@types/node": "10.17.27", - "@types/prettier": "2.6.0", - "jest": "^27.5.1", - "ts-jest": "^27.1.4", - "aws-cdk": "2.80.0", - "ts-node": "^10.9.1", - "typescript": "~3.9.7" - }, - "dependencies": { - "@aws-quickstart/eks-blueprints": "1.6.1", - "aws-cdk-lib": "2.80.0", - "constructs": "^10.0.0", - "simple-base": "^1.0.0", - "source-map-support": "^0.5.21" - } -} diff --git a/analytics/cdk/emr-eks/test/emr-eks-blueprint.test.ts b/analytics/cdk/emr-eks/test/emr-eks-blueprint.test.ts deleted file mode 100644 index b6c4faae7..000000000 --- a/analytics/cdk/emr-eks/test/emr-eks-blueprint.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Tests EMR on EKS AddOn - * - * @group unit/emr-eks-blueprint - */ - -import * as blueprints from '@aws-quickstart/eks-blueprints'; -import * as cdk from 'aws-cdk-lib'; -import { EmrEksAddOn } from '../lib/AddOns/emrEksAddOn'; -import { EmrEksTeam, EmrEksTeamProps } from '../lib/teams/emrEksTeam'; -import { ArnPrincipal, PolicyStatement } from 'aws-cdk-lib/aws-iam'; -import { Match, Template } from 'aws-cdk-lib/assertions'; -import EmrEksStack, { EmrEksBlueprintProps } from '../lib/emr-eks-blueprint-stack'; - -const app = new cdk.App(); - -const account = '123456789012'; -const region = 'eu-west-1'; - -const executionRolePolicyStatement: PolicyStatement[] = [ - new PolicyStatement({ - resources: ['*'], - actions: ['s3:*'], - }), - new PolicyStatement({ - resources: ['*'], - actions: ['glue:*'], - }), - new PolicyStatement({ - resources: ['*'], - actions: [ - 'logs:*', - ], - }), -]; - -const dataTeam: EmrEksTeamProps = { - name: 'dataTeam', - virtualClusterName: 'blueprintjob', - virtualClusterNamespace: 'blueprintjob', - createNamespace: true, - executionRoles: [ - { - executionRoleIamPolicyStatement: executionRolePolicyStatement, - executionRoleName: 'myBlueprintExecRole' - } - ] -}; - -const props: EmrEksBlueprintProps = { env: { account, region }, dataTeams: [dataTeam], clusterAdminRoleArn: new ArnPrincipal('arn:aws:iam::1111111:role/MY-ROLE') }; - -const myStack = new EmrEksStack().build(app, 'AddonRefactotingblueprint', props); - -const template = Template.fromStack(myStack); - -test('Verify the creation of emr-containters service role', () => { - - template.hasResourceProperties('AWS::IAM::ServiceLinkedRole', { - AWSServiceName: "emr-containers.amazonaws.com" - }); - -}); diff --git a/analytics/cdk/emr-eks/test/emr-eks.test.ts b/analytics/cdk/emr-eks/test/emr-eks.test.ts deleted file mode 100644 index b10209e1f..000000000 --- a/analytics/cdk/emr-eks/test/emr-eks.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Tests EMR on EKS AddOn - * - * @group unit/emr-eks-addon - */ - -import * as blueprints from '@aws-quickstart/eks-blueprints'; -import * as cdk from 'aws-cdk-lib'; -import { EmrEksAddOn } from '../lib/AddOns/emrEksAddOn'; -import {EmrEksTeam, EmrEksTeamProps} from '../lib/teams/emrEksTeam'; -import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; -import { Template } from 'aws-cdk-lib/assertions'; - -const app = new cdk.App(); - -const executionRolePolicyStatement: PolicyStatement [] = [ - new PolicyStatement({ - resources: ['*'], - actions: ['s3:*'], - }), - new PolicyStatement({ - resources: ['*'], - actions: ['glue:*'], - }), - new PolicyStatement({ - resources: ['*'], - actions: [ - 'logs:*', - ], - }), - ]; - -const dataTeam: EmrEksTeamProps = { - name:'dataTeam', - virtualClusterName: 'batchJob', - virtualClusterNamespace: 'batchjob', - createNamespace: true, - executionRoles: [ - { - executionRoleIamPolicyStatement: executionRolePolicyStatement, - executionRoleName: 'myBlueprintExecRole' - } - ] - }; - -const parentStack = blueprints.EksBlueprint.builder() - .addOns( - new blueprints.VpcCniAddOn(), - new blueprints.CoreDnsAddOn(), - new blueprints.MetricsServerAddOn, - new blueprints.ClusterAutoScalerAddOn, - new blueprints.CertManagerAddOn, - new blueprints.AwsLoadBalancerControllerAddOn, - new blueprints.EbsCsiDriverAddOn, - new blueprints.KubeProxyAddOn, - new EmrEksAddOn) - .teams( - new EmrEksTeam(dataTeam)) - .build(app, "test-emr-eks-blueprint"); - -const template = Template.fromStack(parentStack); - -test('Verify the creation of emr-containters service role', () => { - - template.hasResourceProperties('AWS::IAM::ServiceLinkedRole', { - AWSServiceName: "emr-containers.amazonaws.com" - } - ); - -}); diff --git a/analytics/cdk/emr-eks/tsconfig.json b/analytics/cdk/emr-eks/tsconfig.json deleted file mode 100644 index ccd3f6a39..000000000 --- a/analytics/cdk/emr-eks/tsconfig.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2018", - "module": "commonjs", - "outDir": "dist", - "lib": [ - "es2018", - "dom" - ], - "declaration": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "noImplicitThis": true, - "alwaysStrict": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": false, - "inlineSourceMap": true, - "inlineSources": true, - "experimentalDecorators": true, - "strictPropertyInitialization": false, - "typeRoots": [ - "./node_modules/@types" - ] - }, - "exclude": [ - "cdk.out", - "dist", - "node_modules", - "test", - ".git", - ".idea", - ".vscode", - "cdk.context.json" - ] -} diff --git a/analytics/cdk/stream-emr-on-eks/.gitignore b/analytics/cdk/stream-emr-on-eks/.gitignore deleted file mode 100644 index a31e3f1c2..000000000 --- a/analytics/cdk/stream-emr-on-eks/.gitignore +++ /dev/null @@ -1,41 +0,0 @@ -# Git -.git -Config - -### VisualStudioCode ### -.vscode/* -### IntelliJ/ PyCharm ### -.idea/* -# System Files -**/.DS_Store -# CDK -**/cdk.out -**/cdk.context.json -*.swp -**/node_modules -**/package-lock.json - -# compiled output -**/global-s3-assets -**/regional-s3-assets -**/open-source - -### Python ### -# Byte-compiled / optimized / DLL files -*__pycache__/ -*.py[cod] -*$py.class -# Python Distribution / packaging -*.egg-info/ -*.egg -# Python Virtual Environments -**/venv* -**/.venv* -**/.env -## Python Testing -.pytest_cache -**/.pytest_cache -**/.coverage -**/.coveragerc -**/coverage-reports/ -*.jar diff --git a/analytics/cdk/stream-emr-on-eks/CODE_OF_CONDUCT.md b/analytics/cdk/stream-emr-on-eks/CODE_OF_CONDUCT.md deleted file mode 100644 index 5b627cfa6..000000000 --- a/analytics/cdk/stream-emr-on-eks/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,4 +0,0 @@ -## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. diff --git a/analytics/cdk/stream-emr-on-eks/CONTRIBUTING.md b/analytics/cdk/stream-emr-on-eks/CONTRIBUTING.md deleted file mode 100644 index c4b6a1c50..000000000 --- a/analytics/cdk/stream-emr-on-eks/CONTRIBUTING.md +++ /dev/null @@ -1,59 +0,0 @@ -# Contributing Guidelines - -Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional -documentation, we greatly value feedback and contributions from our community. - -Please read through this document before submitting any issues or pull requests to ensure we have all the necessary -information to effectively respond to your bug report or contribution. - - -## Reporting Bugs/Feature Requests - -We welcome you to use the GitHub issue tracker to report bugs or suggest features. - -When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already -reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: - -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment - - -## Contributing via Pull Requests -Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: - -1. You are working against the latest source on the *main* branch. -2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. -3. You open an issue to discuss any significant work - we would hate for your time to be wasted. - -To send us a pull request, please: - -1. Fork the repository. -2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass. -4. Commit to your fork using clear commit messages. -5. Send us a pull request, answering any default questions in the pull request interface. -6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. - -GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and -[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). - - -## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. - - -## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. - - -## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. - - -## Licensing - -See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/analytics/cdk/stream-emr-on-eks/Dockerfile b/analytics/cdk/stream-emr-on-eks/Dockerfile deleted file mode 100644 index c5db9916b..000000000 --- a/analytics/cdk/stream-emr-on-eks/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM 895885662937.dkr.ecr.us-west-2.amazonaws.com/spark/emr-6.5.0:latest -USER root -RUN pip3 install --upgrade boto3 pandas numpy -COPY spark-sql-kinesis_2.12-1.2.0_spark-3.0.jar ${SPARK_HOME}/jars/ -USER hadoop:hadoop diff --git a/analytics/cdk/stream-emr-on-eks/LICENSE b/analytics/cdk/stream-emr-on-eks/LICENSE deleted file mode 100644 index 6aa0c458f..000000000 --- a/analytics/cdk/stream-emr-on-eks/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/analytics/cdk/stream-emr-on-eks/README.md b/analytics/cdk/stream-emr-on-eks/README.md deleted file mode 100644 index 504559077..000000000 --- a/analytics/cdk/stream-emr-on-eks/README.md +++ /dev/null @@ -1,340 +0,0 @@ -# Spark Structured Streaming Demo with EMR on EKS - -This is a project developed in Python [CDK](https://docs.aws.amazon.com/cdk/latest/guide/home.html). -It includes sample data, Kafka producer simulator, and a consumer example that can be run with EMR on EC2 or EMR on EKS. Additionally, we have added few Kinesis examples for difference use cases. - -The infrastructure deployment includes the following: -- A new S3 bucket to store sample data and stream job code -- An EKS cluster v1.24 in a new VPC across 2 AZs - - The Cluster has 2 default managed node groups: the OnDemand nodegroup scales from 1 to 5, SPOT instance nodegroup can scale from 1 to 30. - - It also has a Fargate profile labelled with the value `serverless` -- An EMR virtual cluster in the same VPC - - The virtual cluster links to `emr` namespace - - The namespace accommodates two types of Spark jobs, ie. run on managed node group or serverless job on Fargate - - All EMR on EKS configuration are done, including fine-grained access controls for pods by the AWS native solution IAM roles for service accounts -- A MSK Cluster in the same VPC with 2 brokers in total. Kafka version is 2.8.1 - - A Cloud9 IDE as the command line environment in the demo. - - Kafka Client tool will be installed on the Cloud9 IDE -- An EMR on EC2 cluster with managed scaling enabled. - - 1 primary and 1 core nodes with r5.xlarge. - - configured to run one Spark job at a time. - - can scale from 1 to 10 core + task nodes - - mounted EFS for checkpointing test/demo (a bootstrap action) - -## Spark examples - read stream from MSK -Spark consumer applications reading from Amazon MSK: - -* [1. Run a job with EMR on EKS](#1-submit-a-job-with-emr-on-eks) -* [2. Same job with Fargate on EMR on EKS](#2-EMR-on-EKS-with-Fargate) -* [3. Same job with EMR on EC2](#3-optional-Submit-step-to-EMR-on-EC2) - -## Spark examples - read stream from Kinesis -* [1. (Optional) Build a custom docker image](#1-optional-Build-custom-docker-image) -* [2. Run a job with kinesis-sql connector](#2-Use-kinesis-sql-connector) -* [3. Run a job with Spark's DStream](#3-use-spark-s-dstream) - -## Deploy Infrastructure - -The provisioning takes about 30 minutes to complete. -Two ways to deploy: -1. AWS CloudFormation template (CFN) -2. [AWS Cloud Development Kit (AWS CDK)](https://docs.aws.amazon.com/cdk/latest/guide/home.html). - -### CloudFormation Deployment - - | Region | Launch Template | - | --------------------------- | ----------------------- | - | --------------------------- | ----------------------- | - **US East (N. Virginia)**| [![Deploy to AWS](source/app_resources/00-deploy-to-aws.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?stackName=emr-stream-demo&templateURL=https://blogpost-sparkoneks-us-east-1.s3.amazonaws.com/emr-stream-demo/v2.0.0/emr-stream-demo.template) - -* To launch in a different AWS Region, check out the following customization section, or use the CDK deployment option. - -### Customization -You can customize the solution, such as set to a different region, then generate the CFN templates in your required region: -```bash -export BUCKET_NAME_PREFIX= # bucket where customized code will reside -export AWS_REGION= -export SOLUTION_NAME=emr-stream-demo -export VERSION=v2.0.0 # version number for the customized code - -./deployment/build-s3-dist.sh $BUCKET_NAME_PREFIX $SOLUTION_NAME $VERSION - -# create the bucket where customized code will reside -aws s3 mb s3://$BUCKET_NAME_PREFIX-$AWS_REGION --region $AWS_REGION - -# Upload deployment assets to the S3 bucket -aws s3 cp ./deployment/global-s3-assets/ s3://$BUCKET_NAME_PREFIX-$AWS_REGION/$SOLUTION_NAME/$VERSION/ --recursive --acl bucket-owner-full-control -aws s3 cp ./deployment/regional-s3-assets/ s3://$BUCKET_NAME_PREFIX-$AWS_REGION/$SOLUTION_NAME/$VERSION/ --recursive --acl bucket-owner-full-control - -echo -e "\nIn web browser, paste the URL to launch the template: https://console.aws.amazon.com/cloudformation/home?region=$AWS_REGION#/stacks/quickcreate?stackName=emr-stream-demo&templateURL=https://$BUCKET_NAME_PREFIX-$AWS_REGION.s3.amazonaws.com/$SOLUTION_NAME/$VERSION/emr-stream-demo.template\n" -``` - -### CDK Deployment - -#### Prerequisites -Install the following tools: -1. [Python 3.6 +](https://www.python.org/downloads/). -2. [Node.js 10.3.0 +](https://nodejs.org/en/) -3. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-macos.html#install-macosos-bundled). Configure the CLI by `aws configure`. -4. [CDK toolkit](https://cdkworkshop.com/15-prerequisites/500-toolkit.html) -5. [One-off CDK bootstrap](https://cdkworkshop.com/20-typescript/20-create-project/500-deploy.html) for the first time deployment. - -#### Deploy -```bash -python3 -m venv .env -source .env/bin/activate -pip install -r requirements.txt - -cdk deploy -``` - -## Post-deployment - -The following `post-deployment.sh` is executable in Linux, not for Mac OSX. Modify the script if needed. - -1. Open the "Kafka Client" IDE in Cloud9 console. Create one if the Cloud9 IDE doesn't exist. -``` -VPC prefix: 'emr-stream-demo' -Instance Type: 't3.small' -``` -2. [Attach the IAM role that contains `Cloud9Admin` to your IDE](https://catalog.us-east-1.prod.workshops.aws/workshops/d90c2f2d-a84b-4e80-b4f9-f5cee0614426/en-US/30-emr-serverless/31-set-up-env#setup-cloud9-ide). - -3. Turn off AWS managed temporary credentials in Cloud9: -```bash -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -unzip awscliv2.zip -sudo ./aws/install --update -/usr/local/bin/aws cloud9 update-environment --environment-id $C9_PID --managed-credentials-action DISABLE -rm -vf ${HOME}/.aws/credentials -``` - -4. Run the script to configure the cloud9 IDE environment: -```bash -curl https://raw.githubusercontent.com/aws-samples/stream-emr-on-eks/main/deployment/app_code/post-deployment.sh | bash -``` -5. Wait for 5 mins, then check the [MSK cluster](https://console.aws.amazon.com/msk/) status. Make sure it is `active` before sending data to the cluster. -6. Launching a new terminal window in Cloud9, send the sample data to MSK: -```bash -wget https://github.com/xuite627/workshop_flink1015-1/raw/master/dataset/nycTaxiRides.gz -zcat nycTaxiRides.gz | split -l 10000 --filter="kafka_2.12-2.8.1/bin/kafka-console-producer.sh --broker-list ${MSK_SERVER} --topic taxirides ; sleep 0.2" > /dev/null -``` -6. Launching the 3rd terminal window and monitor the source MSK topic: -```bash -kafka_2.12-2.8.1/bin/kafka-console-consumer.sh \ ---bootstrap-server ${MSK_SERVER} \ ---topic taxirides \ ---from-beginning -``` - -## MSK integration -### 1. Submit a job with EMR on EKS - -- [Sample job](deployment/app_code/job/msk_consumer.py) to consume data stream in MSK -- Submit the job: -```bash -aws emr-containers start-job-run \ ---virtual-cluster-id $VIRTUAL_CLUSTER_ID \ ---name msk_consumer \ ---execution-role-arn $EMR_ROLE_ARN \ ---release-label emr-5.33.0-latest \ ---job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "s3://'$S3BUCKET'/app_code/job/msk_consumer.py", - "entryPointArguments":["'$MSK_SERVER'","s3://'$S3BUCKET'/stream/checkpoint/emreks","emreks_output"], - "sparkSubmitParameters": "--conf spark.jars.packages=org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.7 --conf spark.cleaner.referenceTracking.cleanCheckpoints=true --conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.driver.memory=2G --conf spark.executor.cores=2"}}' \ ---configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.kubernetes.driver.podTemplateFile":"s3://'$S3BUCKET'/app_code/job/driver_template.yaml","spark.kubernetes.executor.podTemplateFile":"s3://'$S3BUCKET'/app_code/job/executor_template.yaml" - } - } - ], - "monitoringConfiguration": { - "s3MonitoringConfiguration": {"logUri": "s3://'${S3BUCKET}'/elasticmapreduce/emreks-log/"}} -}' -``` -### Verify the job is running: -```bash -# can see the job pod in EKS -kubectl get po -n emr - -# verify in EMR console -# in Cloud9, run the consumer tool to check if any data comeing through in the target Kafka topic -kafka_2.12-2.8.1/bin/kafka-console-consumer.sh --bootstrap-server ${MSK_SERVER} --topic emreks_output --from-beginning -``` -### Cancel the long-running job (can get job id from the job submission output or in EMR console) -```bash -aws emr-containers cancel-job-run --virtual-cluster-id $VIRTUAL_CLUSTER_ID --id -``` - -### 2. EMR on EKS with Fargate -Run the [same job](deployment/app_code/job/msk_consumer.py) on the same EKS cluster, but with the serverless option - Fargate compute choice. - -To ensure it is picked up by Fargate not by the managed nodegroup on EC2, we will tag the Spark job by a `serverless` label, which has setup in a Fargate profile previously: -```yaml ---conf spark.kubernetes.driver.label.type=serverless ---conf spark.kubernetes.executor.label.type=serverless -``` - -Submit the job to Fargate: - -```bash -aws emr-containers start-job-run \ ---virtual-cluster-id $VIRTUAL_CLUSTER_ID \ ---name msk_consumer_fg \ ---execution-role-arn $EMR_ROLE_ARN \ ---release-label emr-5.33.0-latest \ ---job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "s3://'$S3BUCKET'/app_code/job/msk_consumer.py", - "entryPointArguments":["'$MSK_SERVER'","s3://'$S3BUCKET'/stream/checkpoint/emreksfg","emreksfg_output"], - "sparkSubmitParameters": "--conf spark.jars.packages=org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.7 --conf spark.cleaner.referenceTracking.cleanCheckpoints=true --conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.driver.memory=2G --conf spark.executor.cores=2 --conf spark.kubernetes.driver.label.type=serverless --conf spark.kubernetes.executor.label.type=serverless"}}' \ ---configuration-overrides '{ - "monitoringConfiguration": { - "s3MonitoringConfiguration": {"logUri": "s3://'${S3BUCKET}'/elasticmapreduce/emreksfg-log/"}}}' -``` -### Verify the job is running on EKS Fargate -```bash -kubectl get po -n emr - -# verify in EMR console -# in Cloud9, run the consumer tool to check if any data comeing through in the target Kafka topic -kafka_2.12-2.8.1/bin/kafka-console-consumer.sh \ ---bootstrap-server ${MSK_SERVER} \ ---topic emreksfg_output \ ---from-beginning -``` - -### 3. (Optional) Submit step to EMR on EC2 - -```bash -cluster_id=$(aws emr list-clusters --cluster-states WAITING --query 'Clusters[?Name==`emr-stream-demo`].Id' --output text) -MSK_SERVER=$(echo $MSK_SERVER | cut -d',' -f 2) - -aws emr add-steps \ ---cluster-id $cluster_id \ ---steps Type=spark,Name=emrec2_stream,Args=[--deploy-mode,cluster,--conf,spark.cleaner.referenceTracking.cleanCheckpoints=true,--conf,spark.executor.instances=2,--conf,spark.executor.memory=2G,--conf,spark.driver.memory=2G,--conf,spark.executor.cores=2,--packages,org.apache.spark:spark-sql-kafka-0-10_2.12:3.0.1,s3://$S3BUCKET/app_code/job/msk_consumer.py,$MSK_SERVER,s3://$S3BUCKET/stream/checkpoint/emrec2,emrec2_output],ActionOnFailure=CONTINUE -``` - -### Verify -```bash -# verify in EMR console -# in Cloud9, run the consumer tool to check if any data comeing through in the target Kafka topic -kafka_2.12-2.8.1/bin/kafka-console-consumer.sh \ ---bootstrap-server ${MSK_SERVER} \ ---topic emrec2_output \ ---from-beginning -``` - -## Kinesis integration - -### 1. (Optional) Build custom docker image -We will create & delete a kinesis test stream on the fly via boto3, so a custom EMR on EKS docker image to include the Python library is needed. The custom docker image is not compulsory, if you don't need the boto3 and kinesis-sql connector. - -Build a image based on EMR on EKS 6.5: -```bash -export AWS_REGION=$(aws configure list | grep region | awk '{print $2}') -export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account) -export ECR_URL=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com - -aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 895885662937.dkr.ecr.us-west-2.amazonaws.com -docker build -t emr6.5_custom . - -# create ECR repo in current account -aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $ECR_URL -aws ecr create-repository --repository-name emr6.5_custom_boto3 --image-scanning-configuration scanOnPush=true --region $AWS_REGION - -# push to ECR -docker tag emr6.5_custom $ECR_URL/emr6.5_custom_boto3 -docker push $ECR_URL/emr6.5_custom_boto3 -``` - -### 2. Use kinesis-sql connector -This demo uses the `com.qubole.spark/spark-sql-kinesis_2.12/1.2.0-spark_3.0` connector to interact with Kinesis. - -To enable the job-level access control, ie. the [IRSA feature](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html), we have forked the [kinesis-sql git repo](https://github.com/aws-samples/kinesis-sql) and recompiled a new jar after upgraded the AWS java SDK. The custom docker build above will pick up the upgraded connector automatically. - -- [Sample job](deployment/app_code/job/qubole-kinesis.py) to consume data stream in Kinesis -- Submit the job: -```bash -export AWS_REGION=$(aws configure list | grep region | awk '{print $2}') -export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account) -export ECR_URL=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com - -aws emr-containers start-job-run \ ---virtual-cluster-id $VIRTUAL_CLUSTER_ID \ ---name kinesis-demo \ ---execution-role-arn $EMR_ROLE_ARN \ ---release-label emr-6.5.0-latest \ ---job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "s3://'$S3BUCKET'/app_code/job/qubole-kinesis.py", - "entryPointArguments":["'${AWS_REGION}'","s3://'${S3BUCKET}'/qubolecheckpoint","s3://'${S3BUCKET}'/qubole-kinesis-output"], - "sparkSubmitParameters": "--conf spark.cleaner.referenceTracking.cleanCheckpoints=true"}}' \ ---configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.kubernetes.container.image": "'${ECR_URL}'/emr6.5_custom_boto3:latest" - } - } - ], - "monitoringConfiguration": { - "s3MonitoringConfiguration": {"logUri": "s3://'${S3BUCKET}'/elasticmapreduce/kinesis-fargate-log/"} - } -}' -``` - -### 3. Use Spark's DStream - -This demo uses the `spark-streaming-kinesis-asl_2.12` library to read from Kinesis. Check out the [Spark's official document](https://spark.apache.org/docs/latest/streaming-kinesis-integration.html). The Spark syntax is slightly different from the spark-sql-kinesis approach. It operates at RDD level. - -- [Sample job](deployment/app_code/job/pyspark-kinesis.py) to consume data stream from Kinesis -- Submit the job: -```bash -export AWS_REGION=$(aws configure list | grep region | awk '{print $2}') -export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account) -export ECR_URL=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com - -aws emr-containers start-job-run \ ---virtual-cluster-id $VIRTUAL_CLUSTER_ID \ ---name kinesis-demo \ ---execution-role-arn $EMR_ROLE_ARN \ ---release-label emr-6.5.0-latest \ ---job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "s3://'$S3BUCKET'/app_code/job/pyspark-kinesis.py", - "entryPointArguments":["'${AWS_REGION}'","s3://'$S3BUCKET'/asloutput/"], - "sparkSubmitParameters": "--jars https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kinesis-asl_2.12/3.1.2/spark-streaming-kinesis-asl_2.12-3.1.2.jar,https://repo1.maven.org/maven2/com/amazonaws/amazon-kinesis-client/1.12.0/amazon-kinesis-client-1.12.0.jar"}}' \ ---configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.kubernetes.container.image": "'${ECR_URL}'/emr6.5_custom_boto3:latest" - } - } - ], - "monitoringConfiguration": { - "s3MonitoringConfiguration": {"logUri": "s3://'${S3BUCKET}'/elasticmapreduce/kinesis-fargate-log/"} - } -}' -``` - -## Useful commands - - * `kubectl get pod -n emr` list running Spark jobs - * `kubectl delete pod --all -n emr` delete all Spark jobs - * `kubectl logs -n emr` check logs against a pod in the emr namespace - * `kubectl get node --label-columns=eks.amazonaws.com/capacityType,topology.kubernetes.io/zone` check EKS compute capacity types and AZ distribution. - - -## Clean up -Run the clean-up script with: -```bash -curl https://raw.githubusercontent.com/aws-samples/stream-emr-on-eks/main/deployment/app_code/delete_all.sh | bash -``` -Go to the [CloudFormation console](https://console.aws.amazon.com/cloudformation/home?region=us-east-1), manually delete the remaining resources if needed. diff --git a/analytics/cdk/stream-emr-on-eks/app.py b/analytics/cdk/stream-emr-on-eks/app.py deleted file mode 100644 index 1c88982d0..000000000 --- a/analytics/cdk/stream-emr-on-eks/app.py +++ /dev/null @@ -1,35 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) -#!/usr/bin/env python3 -from aws_cdk import (App,Tags,CfnOutput,Aws) -from source.lib.emr_on_ec2_stack import EMREC2Stack -from source.lib.msk_stack import MSKStack -from source.lib.spark_on_eks_stack import SparkOnEksStack - -app = App() -proj_name = app.node.try_get_context('project_name') -emr_release_v=app.node.try_get_context('emr_version') - -# main stacks -eks_stack = SparkOnEksStack(app, proj_name, proj_name) -msk_stack = MSKStack(eks_stack,'kafka', proj_name, eks_stack.eksvpc) - -# OPTIONAL: nested stack to setup EMR on EC2 -emr_ec2_stack = EMREC2Stack(eks_stack, 'emr-on-ec2', emr_release_v, proj_name, eks_stack.eksvpc, eks_stack.code_bucket) - -Tags.of(eks_stack).add('project', proj_name) -Tags.of(msk_stack).add('project', proj_name) -Tags.of(emr_ec2_stack).add('for-use-with-amazon-emr-managed-policies', 'true') - -# Deployment Output -CfnOutput(eks_stack,'CODE_BUCKET', value=eks_stack.code_bucket) -CfnOutput(eks_stack,"MSK_CLIENT_URL", - value=f"https://{Aws.REGION}.console.aws.amazon.com/cloud9/home/environments/{msk_stack.Cloud9URL}?permissions=owner", - description="Cloud9 Url, Use this URL to access your command line environment in a browser" -) -CfnOutput(eks_stack, "MSK_BROKER", value=msk_stack.MSKBroker) -CfnOutput(eks_stack, "VirtualClusterId",value=eks_stack.EMRVC) -CfnOutput(eks_stack, "FargateVirtualClusterId",value=eks_stack.EMRFargateVC) -CfnOutput(eks_stack, "EMRExecRoleARN", value=eks_stack.EMRExecRole) - -app.synth() diff --git a/analytics/cdk/stream-emr-on-eks/cdk.json b/analytics/cdk/stream-emr-on-eks/cdk.json deleted file mode 100644 index 73b7d21fa..000000000 --- a/analytics/cdk/stream-emr-on-eks/cdk.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "app": "python3 app.py", - "context": { - "project_name": "emr-stream-demo", - "emr_version":"emr-6.9.0", - "version": "v2.0.0", - "@aws-cdk/core:stackRelativeExports": "false" - } -} diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/delete_all.sh b/analytics/cdk/stream-emr-on-eks/deployment/app_code/delete_all.sh deleted file mode 100755 index 4adea66e5..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/delete_all.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -export stack_name="${1:-emr-stream-demo}" - -# delete EMR virtual cluster if needed -emr_cv=$(aws emr-containers list-virtual-clusters --state ARRESTED --query 'virtualClusters[*].id' --output text) -if [ ! -z "$emr_cv" ]; then - for i in emr_cv; do - aws emr-containers delete-virtual-cluster --id $i - done -fi - -# delete S3 -S3BUCKET=$(aws cloudformation describe-stacks --stack-name $stack_name --query "Stacks[0].Outputs[?OutputKey=='CODEBUCKET'].OutputValue" --output text) -if [ ! "$S3BUCKET" == 'None' ]; then - echo "Delete EMR log from S3" - aws s3 rm s3://$S3BUCKET --recursive - aws s3 rb s3://$S3BUCKET --force -fi - -# delete the rest from CF -echo "Delete the rest of resources by CloudFormation delete command" -aws cloudformation delete-stack --stack-name $stack_name diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/driver_template.yaml b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/driver_template.yaml deleted file mode 100644 index 7b94e0a15..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/driver_template.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Pod -spec: - restartPolicy: Always - nodeSelector: - eks.amazonaws.com/capacityType: ON_DEMAND diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/emr-mount-efs.sh b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/emr-mount-efs.sh deleted file mode 100644 index 66393226e..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/emr-mount-efs.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -ex - -if [[ -z "$1" || -z "$2" ]] - then - echo "Missing mandatory arguments: File system ID, region" - exit 1 -fi - -# get file system id from input argument -fs_id=$1 - -# get region from input argument -region_id=$2 - -# verify file system is ready -times=0 -echo -while [ 5 -gt $times ] && ! aws efs describe-file-systems --file-system-id $fs_id --region $region_id --no-paginate | grep -Po "available" -do - sleep 5 - times=$(( $times + 1 )) - echo Attempt $times at verifying efs $fs_id is available... -done - -# verify mount target is ready -times=0 -echo -while [ 5 -gt $times ] && ! aws efs describe-mount-targets --file-system-id $fs_id --region $region_id --no-paginate | grep -Po "available" -do - sleep 5 - times=$(( $times + 1 )) - echo Attempt $times at verifying efs $fs_id mount target is available... -done - -# create local path to mount efs -sudo mkdir -p /efs - -# mount efs -until sudo mount -t nfs4 \ - -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \ - $fs_id.efs.$region_id.amazonaws.com:/ \ - /efs; do echo "Shared filesystem no ready yet..." ; sleep 5; done - -cd /efs - -# give hadoop user permission to efs directory -sudo chown -R hadoop:hadoop . - -if grep $fs_id /proc/mounts; then - echo "File system is mounted successfully." -else - echo "File system mounting is unsuccessful." - exit 1 -fi diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/executor_template.yaml b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/executor_template.yaml deleted file mode 100644 index 47e09a352..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/executor_template.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: Pod -spec: - nodeSelector: - eks.amazonaws.com/capacityType: SPOT diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/msk_consumer.py b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/msk_consumer.py deleted file mode 100644 index 643d37033..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/msk_consumer.py +++ /dev/null @@ -1,62 +0,0 @@ -from pyspark.sql import SparkSession -from pyspark.sql.types import * -from pyspark.sql.functions import * -import pyspark -import sys - -spark = SparkSession.builder \ - .appName("Spark Structured Streaming from Kafka") \ - .getOrCreate() - -sdfRides = spark \ - .readStream \ - .format("kafka") \ - .option("kafka.bootstrap.servers", sys.argv[1]) \ - .option("subscribe", "taxirides") \ - .option("startingOffsets", "latest") \ - .option("auto.offset.reset", "latest") \ - .load() \ - .selectExpr("decode(CAST(value AS STRING),'utf-8') as value") - -taxiRidesSchema = StructType([ \ - StructField("rideId", LongType()), StructField("isStart", StringType()), \ - StructField("endTime", TimestampType()), StructField("startTime", TimestampType()), \ - StructField("startLon", FloatType()), StructField("startLat", FloatType()), \ - StructField("endLon", FloatType()), StructField("endLat", FloatType()), \ - StructField("passengerCnt", ShortType()), StructField("taxiId", LongType()), \ - StructField("driverId", LongType()),StructField("timestamp", TimestampType())]) - -def parse_data_from_kafka_message(sdf, schema): - assert sdf.isStreaming == True, "DataFrame doesn't receive streaming data" - col = split(sdf['value'], ',') - for idx, field in enumerate(schema): - sdf = sdf.withColumn(field.name, col.getItem(idx).cast(field.dataType)) - if field.name=="timestamp": - sdf = sdf.withColumn(field.name, current_timestamp()) - return sdf.select([field.name for field in schema]) - -sdfRides = parse_data_from_kafka_message(sdfRides, taxiRidesSchema) -# sdfFares = parse_data_from_kafka_message(sdfFares, taxiFaresSchema) - -query = sdfRides.withWatermark("timestamp", "10 seconds") \ - .groupBy("driverId", window("timestamp", "10 seconds", "5 seconds")).count() - -# query.writeStream \ -# .outputMode("append") \ -# .format("console") \ -# .option("checkpointLocation", "s3://test/stream/checkpoint/consumer_taxi2") \ -# .option("truncate", False) \ -# .start() \ -# .awaitTermination() - -output=query.select(to_json(struct("*")).alias("value")) \ - .selectExpr("CAST(value AS STRING)") \ - .writeStream \ - .outputMode("append") \ - .format("kafka") \ - .option("kafka.bootstrap.servers", sys.argv[1]) \ - .option("topic", sys.argv[3]) \ - .option("checkpointLocation", sys.argv[2]) \ - .start() - -output.awaitTermination() diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.ipynb b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.ipynb deleted file mode 100644 index 7b3bb025f..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.ipynb +++ /dev/null @@ -1,205 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install boto3" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "os.environ['PYSPARK_SUBMIT_ARGS'] = '--jars https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kinesis-asl_2.12/3.1.2/spark-streaming-kinesis-asl_2.12-3.1.2.jar,https://repo1.maven.org/maven2/com/amazonaws/amazon-kinesis-client/1.14.8/amazon-kinesis-client-1.14.8.jar,https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-kinesis/1.12.31/aws-java-sdk-kinesis-1.12.31.jar,/home/jovyan/.local/lib/python3.7/site-packages pyspark-shell'\n", - "\n", - "from pyspark import SparkContext\n", - "from pyspark.streaming import StreamingContext\n", - "from pyspark.sql.types import StructField, StructType, StringType, IntegerType\n", - "from pyspark.streaming.kinesis import KinesisUtils, InitialPositionInStream\n", - "import boto3,json" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "create a new stream\n", - "the stream exists\n" - ] - } - ], - "source": [ - "# make sure you have your AWS cerdentials set before you can run this\n", - "\n", - "# creating the Kinesis stream\n", - "client = boto3.client('kinesis')\n", - "stream_name='pyspark-kinesis'\n", - "\n", - "try:\n", - " print(\"create a new stream\")\n", - " client.create_stream(\n", - " StreamName=stream_name,\n", - " ShardCount=1)\n", - "except:\n", - " print(\"the stream exists\")" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [], - "source": [ - "# creating a couple of messages to send to kinesis\n", - "messages = [\n", - " {'message_type': 'message1', 'count': 2},\n", - " {'message_type': 'message2', 'count': 1},\n", - " {'message_type': 'message1', 'count': 2},\n", - " {'message_type': 'message3', 'count': 3},\n", - " {'message_type': 'message1', 'count': 5}\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [], - "source": [ - "for message in messages:\n", - " client.put_record(\n", - " StreamName=stream_name,\n", - " Data=json.dumps(message),\n", - " PartitionKey='part_key')" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [], - "source": [ - "sc = SparkContext(appName=\"PythonStreamingKinesisWordCountAsl\")\n", - "ssc = StreamingContext(sc, 1)" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [], - "source": [ - "kinesis = KinesisUtils.createStream(\n", - " ssc, stream_name,stream_name, 'https://kinesis.us-east-1.amazonaws.com','us-east-1', \\\n", - " InitialPositionInStream.TRIM_HORIZON, 2)" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def format_sample(x):\n", - " data = json.loads(x)\n", - " return (data[0], json.dumps(data))" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "parsed = kinesis.map(lambda x: format_sample(x))\n", - "parsed.pprint()" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "-------------------------------------------\n", - "Time: 2022-03-21 19:05:15\n", - "-------------------------------------------\n", - "\n", - "-------------------------------------------\n", - "Time: 2022-03-21 19:05:16\n", - "-------------------------------------------\n", - "\n", - "-------------------------------------------\n", - "Time: 2022-03-21 19:05:17\n", - "-------------------------------------------\n", - "\n", - "-------------------------------------------\n", - "Time: 2022-03-21 19:05:18\n", - "-------------------------------------------\n", - "\n", - "-------------------------------------------\n", - "Time: 2022-03-21 19:05:19\n", - "-------------------------------------------\n", - "\n" - ] - } - ], - "source": [ - "ssc.start()\n", - "ssc.awaitTerminationOrTimeout(5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# cleanup\n", - "# client.delete_stream(StreamName=stream_name)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.py b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.py deleted file mode 100644 index 9e6fbbeaa..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import print_function - -from pyspark.streaming.kinesis import KinesisUtils, InitialPositionInStream -from pyspark import SparkContext -from pyspark.streaming import StreamingContext -import boto3,json,sys - -# def printRecord(rdd): -# print("========================================================") -# print("Starting new RDD") -# print("========================================================") -# rdd.foreach(lambda record: print(record.encode('utf8'))) - -if __name__ == "__main__": - - # creating the Kinesis stream - stream_name='pyspark-kinesis' - client_region = sys.argv[1] - client = boto3.client('kinesis', client_region) - try: - print("create a new stream") - client.create_stream( - StreamName=stream_name, - ShardCount=1) - except: - print("the stream exists") - # creating a couple of messages to send to kinesis - # messages = [ - # {'message_type': 'message1', 'count': 2}, - # {'message_type': 'message2', 'count': 1}, - # {'message_type': 'message1', 'count': 2}, - # {'message_type': 'message3', 'count': 3}, - # {'message_type': 'message1', 'count': 5} - # ] - - # for message in messages: - # client.put_record( - # StreamName=stream_name, - # Data=json.dumps(message), - # PartitionKey='part_key') - - - # start Spark process, read from kinesis - appName = "PythonStreamingKinesisAsl" - endpointUrl="https://kinesis."+client_region+".amazonaws.com" - sc = SparkContext(appName=appName) - ssc = StreamingContext(sc, 2) - - kinesis = KinesisUtils.createStream(ssc,appName,stream_name,endpointUrl,client_region, InitialPositionInStream.LATEST, 2) - kinesis.pprint() - # # write to s3 - # py_rdd = kinesis.map(lambda x: json.loads(x.encode('utf8'))) - # py_rdd.saveAsTextFiles(sys.argv[2]) - - def format_sample(x): - data = json.loads(x) - return (data['message_type'], json.dumps(data)) - # print to console - parsed = kinesis.map(lambda x: format_sample(x)) - parsed.pprint() - - ssc.start() - ssc.awaitTermination() diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis_submit.sh b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis_submit.sh deleted file mode 100755 index 4f0f77ca8..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/pyspark-kinesis_submit.sh +++ /dev/null @@ -1,27 +0,0 @@ -export AWS_REGION=$(aws configure list | grep region | awk '{print $2}') -export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account) -export ECR_URL=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com - -aws emr-containers start-job-run \ ---virtual-cluster-id $VIRTUAL_CLUSTER_ID \ ---name kinesis-demo \ ---execution-role-arn $EMR_ROLE_ARN \ ---release-label emr-6.5.0-latest \ ---job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "s3://'$S3BUCKET'/app_code/job/pyspark-kinesis.py", - "entryPointArguments":["'${AWS_REGION}'","s3://'$S3BUCKET'/asloutput/"], - "sparkSubmitParameters": "--jars https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kinesis-asl_2.12/3.1.2/spark-streaming-kinesis-asl_2.12-3.1.2.jar,https://repo1.maven.org/maven2/com/amazonaws/amazon-kinesis-client/1.12.0/amazon-kinesis-client-1.12.0.jar"}}' \ ---configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.kubernetes.container.image": "'${ECR_URL}'/emr6.5_custom_boto3:latest" - } - } - ], - "monitoringConfiguration": { - "s3MonitoringConfiguration": {"logUri": "s3://'${S3BUCKET}'/elasticmapreduce/kinesis-fargate-log/"} - } -}' diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.ipynb b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.ipynb deleted file mode 100644 index 889ffaffc..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.ipynb +++ /dev/null @@ -1,174 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install boto3" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "os.environ['PYSPARK_SUBMIT_ARGS'] = '--jars /home/jovyan/spark-sql-kinesis_2.12-1.2.0_spark-3.0-SNAPSHOT.jar,/home/jovyan/.local/lib/python3.7/site-packages pyspark-shell'\n", - "\n", - "from pyspark.sql import SparkSession\n", - "from pyspark.sql.functions import from_json, col\n", - "from pyspark.sql.types import StructField, StructType, StringType, IntegerType\n", - "import boto3\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "create a new stream\n", - "the stream exists\n" - ] - } - ], - "source": [ - "# you don't need any AWS cerdentials set when running with EMR on EKS. We will use IRSA feature to control the access.\n", - "\n", - "# creating the Kinesis stream\n", - "client = boto3.client('kinesis')\n", - "stream_name='pyspark-kinesis'\n", - "\n", - "try:\n", - " print(\"create a new stream\")\n", - " client.create_stream(\n", - " StreamName=stream_name,\n", - " ShardCount=1)\n", - "except:\n", - " print(\"the stream exists\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# creating a couple of messages to send to kinesis\n", - "messages = [\n", - " {'message_type': 'message1', 'count': 2},\n", - " {'message_type': 'message2', 'count': 1},\n", - " {'message_type': 'message1', 'count': 2},\n", - " {'message_type': 'message3', 'count': 3},\n", - " {'message_type': 'message1', 'count': 5}\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "for message in messages:\n", - " client.put_record(\n", - " StreamName=stream_name,\n", - " Data=json.dumps(message),\n", - " PartitionKey='part_key')" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "spark = SparkSession.builder.appName('PySparkKinesis').getOrCreate()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "kinesis = spark \\\n", - " .readStream \\\n", - " .format('kinesis') \\\n", - " .option('streamName', stream_name) \\\n", - " .option('endpointUrl', 'https://kinesis.us-east-1.amazonaws.com')\\\n", - " .option('region', 'us-east-1') \\\n", - " .option('startingposition', 'TRIM_HORIZON')\\\n", - " .option('awsUseInstanceProfile', 'false') \\\n", - " .load()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "schema = StructType([\n", - " StructField(\"message_type\", StringType()),\n", - " StructField(\"count\", IntegerType())])" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "stream = kinesis\\\n", - " .selectExpr('CAST(data AS STRING)')\\\n", - " .select(from_json('data', schema).alias('data'))\\\n", - " .select('data.*')\\\n", - " .writeStream\\\n", - " .outputMode('append')\\\n", - " .format('console')\\\n", - " .trigger(once=True)\\\n", - " .start()\n", - " .awaitTermination()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# delete the kinesis stream\n", - "# client.delete_stream(StreamName=stream_name)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.py b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.py deleted file mode 100644 index 12e7a4112..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis.py +++ /dev/null @@ -1,79 +0,0 @@ -from pyspark.sql import SparkSession -from pyspark.sql.functions import from_json, col -from pyspark.sql.types import StructField, StructType, StringType, IntegerType -import boto3,json,sys - -# creating a Kinesis stream -stream_name='pyspark-kinesis' -client_region = sys.argv[1] -client = boto3.client('kinesis', client_region) - -try: - print("create a new stream") - client.create_stream( - StreamName=stream_name, - ShardCount=1) -except: - print("the stream exists") - -# sending a couple of messages to kinesis -messages = [ - {'message_type': 'message1', 'count': 2}, - {'message_type': 'message2', 'count': 1}, - {'message_type': 'message1', 'count': 2}, - {'message_type': 'message3', 'count': 3}, - {'message_type': 'message1', 'count': 5} -] -for message in messages: - client.put_record( - StreamName=stream_name, - Data=json.dumps(message), - PartitionKey='part_key') - -spark = SparkSession.builder \ - .appName('PySparkKinesis') \ - .getOrCreate() - -# spark.sparkContext.setLogLevel("DEBUG") -kinesis = spark \ - .readStream \ - .format('kinesis') \ - .option('streamName', stream_name) \ - .option('endpointUrl', 'https://kinesis.'+client_region+'.amazonaws.com')\ - .option('region', client_region) \ - .option('startingposition', 'TRIM_HORIZON')\ - .option('awsUseInstanceProfile', 'false') \ - .load() - - -schema = StructType([ - StructField("message_type", StringType()), - StructField("count", IntegerType())]) - - -kinesis.selectExpr('CAST(data AS STRING)')\ - .select(from_json('data', schema).alias('data'))\ - .select('data.*')\ - .writeStream \ - .outputMode('append')\ - .format('console') \ - .trigger(once=True) \ - .start() \ - .awaitTermination() - -# write to s3 -# kinesis.selectExpr('CAST(data AS STRING)')\ -# .select(from_json('data',data_schema).alias('data'))\ -# .select('data.*')\ -# .writeStream \ -# .outputMode('append')\ -# .format('parquet') \ -# .option('truncate', True) \ -# .option('checkpointLocation', sys.argv[2]) \ -# .option('path',sys.argv[3]) \ -# .trigger(processingTime='5 seconds') \ -# .start() \ -# .awaitTermination() - -# delete the kinesis stream -# client.delete_stream(StreamName=stream_name) diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis_submit.sh b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis_submit.sh deleted file mode 100755 index e72d1dd2f..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/qubole-kinesis_submit.sh +++ /dev/null @@ -1,27 +0,0 @@ -export AWS_REGION=$(aws configure list | grep region | awk '{print $2}') -export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account) -export ECR_URL=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com - -aws emr-containers start-job-run \ ---virtual-cluster-id $VIRTUAL_CLUSTER_ID \ ---name kinesis-demo \ ---execution-role-arn $EMR_ROLE_ARN \ ---release-label emr-6.2.0-latest \ ---job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "s3://'$S3BUCKET'/app_code/job/qubole-kinesis.py", - "entryPointArguments":["'${AWS_REGION}'","s3://'${S3BUCKET}'/qubolecheckpoint","s3://'${S3BUCKET}'/qubole-kinesis-output"], - "sparkSubmitParameters": "--conf spark.cleaner.referenceTracking.cleanCheckpoints=true"}}' \ ---configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.kubernetes.container.image": "'${ECR_URL}'/emr6.5_custom_boto3:latest" - } - } - ], - "monitoringConfiguration": { - "s3MonitoringConfiguration": {"logUri": "s3://'${S3BUCKET}'/elasticmapreduce/kinesis-fargate-log/"} - } -}' diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/wordcount.py b/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/wordcount.py deleted file mode 100644 index 62d410a96..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/job/wordcount.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys -from pyspark.sql import SparkSession -spark = SparkSession.builder.appName('Amazon reviews word count').getOrCreate() -df = spark.read.parquet(sys.argv[1]) -df.selectExpr("explode(split(lower(review_body), ' ')) as words").groupBy("words").count().write.mode("overwrite").parquet(sys.argv[2]) -exit() diff --git a/analytics/cdk/stream-emr-on-eks/deployment/app_code/post-deployment.sh b/analytics/cdk/stream-emr-on-eks/deployment/app_code/post-deployment.sh deleted file mode 100755 index 03cbd8336..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/app_code/post-deployment.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -export stack_name="${1:-emr-stream-demo}" - -# 0. Setup AWS environment -echo "Setup AWS environment ..." -sudo yum -y install jq -export AWS_REGION=$(curl http://169.254.169.254/latest/meta-data/placement/region) -export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account) - -echo "export AWS_REGION=${AWS_REGION}" | tee -a ~/.bash_profile -echo "export ACCOUNT_ID=${ACCOUNT_ID}" | tee -a ~/.bash_profile -aws configure set default.region ${AWS_REGION} -aws configure get default.region - -export S3BUCKET=$(aws cloudformation describe-stacks --stack-name $stack_name --query "Stacks[0].Outputs[?OutputKey=='CODEBUCKET'].OutputValue" --output text) -export MSK_SERVER=$(aws cloudformation describe-stacks --stack-name $stack_name --query "Stacks[0].Outputs[?OutputKey=='MSKBROKER'].OutputValue" --output text) -export VIRTUAL_CLUSTER_ID=$(aws cloudformation describe-stacks --stack-name $stack_name --query "Stacks[0].Outputs[?OutputKey=='VirtualClusterId'].OutputValue" --output text) -export EMR_ROLE_ARN=$(aws cloudformation describe-stacks --stack-name $stack_name --query "Stacks[0].Outputs[?OutputKey=='EMRExecRoleARN'].OutputValue" --output text) - -echo "export S3BUCKET=${S3BUCKET}" | tee -a ~/.bash_profile -echo "export MSK_SERVER=${MSK_SERVER}" | tee -a ~/.bash_profile -echo "export VIRTUAL_CLUSTER_ID=${VIRTUAL_CLUSTER_ID}" | tee -a ~/.bash_profile -echo "export EMR_ROLE_ARN=${EMR_ROLE_ARN}" | tee -a ~/.bash_profile -source ~/.bash_profile - -# 1. install k8s command tools -echo "Installing kubectl tool..." -curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.24.10/2023-01-30/bin/linux/amd64/kubectl -chmod +x kubectl -mkdir -p $HOME/bin && mv kubectl $HOME/bin/kubectl && export PATH=$PATH:$HOME/bin - -# 2. Update MSK with custom configuration -base64 <msk-config.txt -auto.create.topics.enable=true -log.retention.minutes=1440 -zookeeper.connection.timeout.ms=1000 -log.roll.ms=60480000 -EoF - -validate=$(aws kafka list-configurations --query 'Configurations[?Name==`autotopic`].Arn' --output text) -if [ -z "$validate" ]; then - echo "Update MSK configuration ..." - - configArn=$(aws kafka create-configuration --name "autotopic" --description "Topic autocreation enabled; Log retention 24h; Apache ZooKeeper timeout 1000 ms; Log rolling 16h." --server-properties file://msk-config.txt | jq -r '.Arn') - msk_cluster=$(aws kafka list-clusters --region $AWS_REGION --query 'ClusterInfoList[?ClusterName==`'$stack_name'`].ClusterArn' --output text) - msk_version=$(aws kafka describe-cluster --cluster-arn ${msk_cluster} --query "ClusterInfo.CurrentVersion" --output text) - aws kafka update-cluster-configuration --cluster-arn ${msk_cluster} --configuration-info '{"Arn": "'$configArn'","Revision": 1 }' --current-version ${msk_version} -fi - -# 3. install Kafka Client -echo "Installing Kafka Client tool ..." -wget https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz -tar -xzf kafka_2.12-2.8.1.tgz -rm kafka_2.12-2.8.1.tgz - -# 4. connect to the EKS newly created -echo $(aws cloudformation describe-stacks --stack-name $stack_name --query "Stacks[0].Outputs[?starts_with(OutputKey,'eksclusterEKSConfig')].OutputValue" --output text) | bash -echo "Testing EKS connection..." -kubectl get svc diff --git a/analytics/cdk/stream-emr-on-eks/deployment/build-s3-dist.sh b/analytics/cdk/stream-emr-on-eks/deployment/build-s3-dist.sh deleted file mode 100755 index d823a28d4..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/build-s3-dist.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/bash -# -# This script packages your project into a solution distributable that can be -# used as an input to the solution builder validation pipeline. -# -# Important notes and prereq's: -# 1. The initialize-repo.sh script must have been run in order for this script to -# function properly. -# 2. This script should be run from the repo's root folder. -# -# This script will perform the following tasks: -# 1. Remove any old dist files from previous runs. -# 2. Install dependencies for the cdk-solution-helper; responsible for -# converting standard 'cdk synth' output into solution assets. -# 3. Build and synthesize your CDK project. -# 4. Run the cdk-solution-helper on template outputs and organize -# those outputs into the /global-s3-assets folder. -# 5. Organize source code artifacts into the /regional-s3-assets folder. -# 6. Remove any temporary files used for staging. -# -# Parameters: -# - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda -# code from. The template will append '-[region_name]' to this bucket name. -# For example: ./build-s3-dist.sh solutions v1.0.0 -# The template will then expect the source code to be located in the solutions-[region_name] bucket -# - solution-name: name of the solution for consistency -# - version-code: version of the package - -# Important: CDK global version number -cdk_version===2.67.0 - -# Check to see if the required parameters have been provided: -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside." - echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0 template-bucket-name" - exit 1 -fi - -# Get reference for all important folders -template_dir="$PWD" -staging_dist_dir="$template_dir/staging" -app_code_dir="$template_dir/deployment/app_code" -template_dist_dir="$template_dir/deployment/global-s3-assets" -build_dist_dir="$template_dir/deployment/regional-s3-assets" - -echo "------------------------------------------------------------------------------" -echo "[Init] Remove any old dist files from previous runs" -echo "------------------------------------------------------------------------------" - -echo "rm -rf $template_dist_dir" -rm -rf $template_dist_dir -echo "mkdir -p $template_dist_dir" -mkdir -p $template_dist_dir -echo "rm -rf $build_dist_dir" -rm -rf $build_dist_dir -echo "mkdir -p $build_dist_dir" -mkdir -p $build_dist_dir -echo "rm -rf $staging_dist_dir" -rm -rf $staging_dist_dir -echo "mkdir -p $staging_dist_dir" -mkdir -p $staging_dist_dir - -echo "------------------------------------------------------------------------------" -echo "[Init] Install dependencies for the cdk-solution-helper" -echo "------------------------------------------------------------------------------" - -echo "cd $template_dir/deployment/cdk-solution-helper" -cd $template_dir/deployment/cdk-solution-helper -echo "npm install" -# npm audit fix --force -npm install - -cd $template_dir -echo "pip3 install -r requirements.txt" -python3 -m venv .env -source .env/bin/activate -pip3 install --upgrade pip -r requirements.txt - -echo "------------------------------------------------------------------------------" -echo "[Synth] CDK Project" -echo "------------------------------------------------------------------------------" - -# # Install the global aws-cdk package -echo "npm install -g aws-cdk@$cdk_version" -# npm audit fix --force -npm install aws-cdk@$cdk_version - -# Run 'cdk synth' to generate raw solution outputs -echo "cdk synth --output=$staging_dist_dir" -node_modules/aws-cdk/bin/cdk synth --output=$staging_dist_dir - -# Remove unnecessary output files -echo "cd $staging_dist_dir" -cd $staging_dist_dir -echo "rm tree.json manifest.json cdk.out" -rm tree.json manifest.json cdk.out - -echo "------------------------------------------------------------------------------" -echo "[Packing] Template artifacts" -echo "------------------------------------------------------------------------------" - -# Move outputs from staging to template_dist_dir -echo "Move outputs from staging to template_dist_dir" -mv $staging_dist_dir/*.json $template_dist_dir/ - -# Rename all *.template.json files to *.template -echo "Rename all *.template.json to *.template" -echo "copy templates and rename" -for f in $template_dist_dir/*.template.json; do - mv -- "$f" "${f%.template.json}.template" -done - -# Run the helper to clean-up the templates and remove unnecessary CDK elements -echo "Run the helper to clean-up the templates and remove unnecessary CDK elements" -echo "node $template_dir/deployment/cdk-solution-helper/index" -node $template_dir/deployment/cdk-solution-helper/index -if [ "$?" = "1" ]; then - echo "(cdk-solution-helper) ERROR: there is likely output above." 1>&2 - exit 1 -fi - -# Find and replace bucket_name, solution_name, and version -echo "Find and replace bucket_name, solution_name, and version" -cd $template_dist_dir -echo "Updating code source bucket in template with $1" -replace="s/%%BUCKET_NAME%%/$1/g" -echo "sed -i '' -e $replace $template_dist_dir/*.template" -sed -i '' -e $replace $template_dist_dir/*.template -replace="s/%%SOLUTION_NAME%%/$2/g" -echo "sed -i '' -e $replace $template_dist_dir/*.template" -sed -i '' -e $replace $template_dist_dir/*.template -replace="s/%%VERSION%%/$3/g" -echo "sed -i '' -e $replace $template_dist_dir/*.template" -sed -i '' -e $replace $template_dist_dir/*.template - -# Put Global and Regional code files in a single bucket if the 4th parameter doesn't exist -if [ -z "$4" ]; then - replace="s/%%TEMPLATE_OUTPUT_BUCKET%%/$1"-"${AWS_REGION}/g" -else - replace="s/%%TEMPLATE_OUTPUT_BUCKET%%/$4/g" -fi - -echo "sed -i '' -e $replace $template_dist_dir/*.template" -sed -i '' -e $replace $template_dist_dir/*.template - -rm $template_dist_dir/*.json - -echo "------------------------------------------------------------------------------" -echo "[Packing] Source code artifacts" -echo "------------------------------------------------------------------------------" - -# General cleanup of node_modules and package-lock.json files -echo "find $staging_dist_dir -iname "node_modules" -type d -exec rm -rf "{}" \; 2> /dev/null" -find $staging_dist_dir -iname "node_modules" -type d -exec rm -rf "{}" \; 2> /dev/null -echo "find $staging_dist_dir -iname "package-lock.json" -type f -exec rm -f "{}" \; 2> /dev/null" -find $staging_dist_dir -iname "package-lock.json" -type f -exec rm -f "{}" \; 2> /dev/null - -# ... For each asset.* source code artifact in the temporary /staging folder... -cd $staging_dist_dir -for d in `find . -mindepth 1 -maxdepth 1 -type d`; do - - # Rename the artifact, removing the period for handler compatibility - pfname="$(basename -- $d)" - fname="$(echo $pfname | sed -e 's/\.//g')" - echo "zip -r $fname.zip $fname" - mv $d $fname - cd $staging_dist_dir/$fname - - # Build the artifacts - if ls *.py 1> /dev/null 2>&1; then - echo "====================================" - echo "This is Python runtime" - echo "====================================" - venv_folder=".venv-prod" - rm -fr .venv-test - rm -fr .venv-prod - echo "Initiating virtual environment" - python3 -m venv $venv_folder - source $venv_folder/bin/activate - pip3 install --upgrade pip -q $template_dir --target $venv_folder/lib/python3.*/site-packages - echo "package python artifact" - cd $venv_folder/lib/python3.*/site-packages - zip -qr9 $staging_dist_dir/$fname.zip . -x "aws_cdk/*" - echo "zip -r $staging_dist_dir/$fname" - cd $staging_dist_dir/$fname - rm -rf $venv_folder - zip -grq $staging_dist_dir/$fname.zip . - - elif ls *.js 1> /dev/null 2>&1; then - echo "====================================" - echo "This is Node runtime" - echo "====================================" - echo "Clean and rebuild artifacts" - # npm audit fix --force - npm run - npm ci - echo "copy package.json and package-lock.json files" - cp -rf $template_dir/deployment/cdk-solution-helper/*.json . - if [ "$?" = "1" ]; then - echo "ERROR: Seems like package-lock.json does not exists or is out of sync with package.josn. Trying npm install instead" 1>&2 - npm install - fi - # Zip the artifact - echo "zip -r $staging_dist_dir/$fname" - zip -qr9 $staging_dist_dir/$fname.zip . - else - # Zip the artifact - echo "zip -r $staging_dist_dir/$fname" - zip -rq $staging_dist_dir/$fname.zip . - fi - - cd $staging_dist_dir - # Copy the zipped artifact from /staging to /regional-s3-assets - echo "cp $fname.zip $build_dist_dir" - mv $fname.zip $build_dist_dir - - # Remove the old, unzipped artifact from /staging - echo "rm -rf $fname" - rm -rf $fname - - # ... repeat until all source code artifacts are zipped and placed in the - # ... /regional-s3-assets folder - -done - -echo "------------------------------------------------------------------------------" -echo "[Move] the zip files from staging to regional-s3-assets folder" -echo "------------------------------------------------------------------------------" -for d in `find . -mindepth 1 -maxdepth 1`; do - pfname="$(basename -- $d)" - fname="$(echo $pfname | sed -e 's/asset./asset/g')" - mv $d $build_dist_dir/$fname -done - -echo "------------------------------------------------------------------------------" -echo "[Cleanup] Remove temporary files" -echo "------------------------------------------------------------------------------" - -# Delete the temporary /staging folder -echo "rm -rf $staging_dist_dir" -rm -rf $staging_dist_dir diff --git a/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/README.md b/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/README.md deleted file mode 100755 index 14031250c..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/README.md +++ /dev/null @@ -1,152 +0,0 @@ -# cdk-solution-helper - -A lightweight helper function that cleans-up synthesized templates from the AWS Cloud Development Kit (CDK) and prepares -them for use with the AWS Solutions publishing pipeline. This function performs the following tasks: - -#### Lambda function preparation - -Replaces the AssetParameter-style properties that identify source code for Lambda functions with the common variables -used by the AWS Solutions publishing pipeline. - -- `Code.S3Bucket` is assigned the `%%BUCKET_NAME%%` placeholder value. -- `Code.S3Key` is assigned the `%%SOLUTION_NAME%%`/`%%VERSION%%` placeholder value. -- `Handler` is given a prefix identical to the artifact hash, enabling the Lambda function to properly find the handler in the extracted source code package. - -These placeholders are then replaced with the appropriate values using the default find/replace operation run by the pipeline. - -Before: -``` -"examplefunction67F55935": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Ref": "AssetParametersd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7S3Bucket54E71A95" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Fn::Select": [ - 0, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParametersd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7S3VersionKeyC789D8B1" - } - ] - } - ] - }, - { - "Fn::Select": [ - 1, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParametersd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7S3VersionKeyC789D8B1" - } - ] - } - ] - } - ] - ] - } - }, ... - Handler: "index.handler", ... -``` - -After helper function run: -``` -"examplefunction67F55935": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": "%%BUCKET_NAME%%", - "S3Key": "%%SOLUTION_NAME%%/%%VERSION%%/assetd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7.zip" - }, ... - "Handler": "assetd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7/index.handler" -``` - -After build script run: -``` -"examplefunction67F55935": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": "solutions", - "S3Key": "trademarked-solution-name/v1.0.0/asset.d513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7.zip" - }, ... - "Handler": "assetd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7/index.handler" -``` - -After CloudFormation deployment: -``` -"examplefunction67F55935": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": "solutions-us-east-1", - "S3Key": "trademarked-solution-name/v1.0.0/asset.d513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7.zip" - }, ... - "Handler": "assetd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7/index.handler" -``` - -#### Template cleanup - -Cleans-up the parameters section and improves readability by removing the AssetParameter-style fields that would have -been used to specify Lambda source code properties. This allows solution-specific parameters to be highlighted and -removes unnecessary clutter. - -Before: -``` -"Parameters": { - "AssetParametersd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7S3Bucket54E71A95": { - "Type": "String", - "Description": "S3 bucket for asset \"d513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7\"" - }, - "AssetParametersd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7S3VersionKeyC789D8B1": { - "Type": "String", - "Description": "S3 key for asset version \"d513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7\"" - }, - "AssetParametersd513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7ArtifactHash7AA751FE": { - "Type": "String", - "Description": "Artifact hash for asset \"d513e93e266931de36e1c7e79c27b196f84ab928fce63d364d9152ca501551f7\"" - }, - "CorsEnabled" : { - "Description" : "Would you like to enable Cross-Origin Resource Sharing (CORS) for the image handler API? Select 'Yes' if so.", - "Default" : "No", - "Type" : "String", - "AllowedValues" : [ "Yes", "No" ] - }, - "CorsOrigin" : { - "Description" : "If you selected 'Yes' above, please specify an origin value here. A wildcard (*) value will support any origin.", - "Default" : "*", - "Type" : "String" - } - } - ``` - -After: -``` -"Parameters": { - "CorsEnabled" : { - "Description" : "Would you like to enable Cross-Origin Resource Sharing (CORS) for the image handler API? Select 'Yes' if so.", - "Default" : "No", - "Type" : "String", - "AllowedValues" : [ "Yes", "No" ] - }, - "CorsOrigin" : { - "Description" : "If you selected 'Yes' above, please specify an origin value here. A wildcard (*) value will support any origin.", - "Default" : "*", - "Type" : "String" - } - } - ``` - -*** -© Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/index.js b/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/index.js deleted file mode 100755 index 7637236ff..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/index.js +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env node -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) -// Imports -const fs = require('fs'); - -// Paths -var currentPath = process.cwd(); -const global_s3_assets = currentPath+'/../deployment/global-s3-assets'; -const solution_name='emr-stream-demo'; - -function setParameter(template) { - const parameters = (template.Parameters) ? template.Parameters : {}; - const assetParameters = Object.keys(parameters).filter(function(key) { - return key.includes('BootstrapVersion'); - }); - assetParameters.forEach(function(a) { - template.Parameters[a] = undefined; - }); - const rules = (template.Rules) ? template.Rules : {}; - const rule = Object.keys(rules).filter(function(key) { - return key.includes('CheckBootstrapVersion'); - }); - rule.forEach(function(a) { - template.Rules[a] = undefined; - }) -} -function assetRef(s3BucketRef) { - // Get S3 bucket key references from assets file - const raw_meta = fs.readFileSync(`${global_s3_assets}/${solution_name}.assets.json`); - let template = JSON.parse(raw_meta); - const metadata = (template.files[s3BucketRef]) ? template.files[s3BucketRef] : {}; - var assetPath = metadata.source.path.replace('.json',''); - return assetPath; -} - -// For each template in global_s3_assets ... -fs.readdirSync(global_s3_assets).forEach(file => { - if ( file != `${solution_name}.assets.json`) { - // Import and parse template file - const raw_template = fs.readFileSync(`${global_s3_assets}/${file}`); - let template = JSON.parse(raw_template); - - //1. Clean-up parameters section - setParameter(template); - - const resources = (template.Resources) ? template.Resources : {}; - //3. Clean-up Account ID and region to enable cross account deployment - const rsrctype=[ - "AWS::Lambda::Function", - "AWS::Lambda::LayerVersion", - "Custom::CDKBucketDeployment", - "AWS::CloudFormation::Stack", - "AWS::CloudFront::Distribution" - ] - const focusTemplate = Object.keys(resources).filter(function(key) { - return (resources[key].Type.indexOf(rsrctype) < 0) - }); - focusTemplate.forEach(function(f) { - const fn = template.Resources[f]; - if (fn.Properties.hasOwnProperty('Code') && fn.Properties.Code.hasOwnProperty('S3Bucket')) { - // Set Lambda::Function S3 reference to regional folder - if (! String(fn.Properties.Code.S3Bucket.Ref).startsWith('appcode')){ - fn.Properties.Code.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/asset`+fn.Properties.Code.S3Key; - fn.Properties.Code.S3Bucket = {'Fn::Sub': '%%BUCKET_NAME%%-${AWS::Region}'}; - } - } - else if (fn.Properties.hasOwnProperty('Content') && fn.Properties.Content.hasOwnProperty('S3Bucket')) { - // Set Lambda::LayerVersion S3 bucket reference - fn.Properties.Content.S3Key = `%%SOLUTION_NAME%%/%%VERSION%%/asset`+fn.Properties.Content.S3Key; - fn.Properties.Content.S3Bucket = {'Fn::Sub': '%%BUCKET_NAME%%-${AWS::Region}'}; - } - else if (fn.Properties.hasOwnProperty('SourceBucketNames')) { - // Set CDKBucketDeployment S3 bucket reference - fn.Properties.SourceObjectKeys = [`%%SOLUTION_NAME%%/%%VERSION%%/asset`+fn.Properties.SourceObjectKeys[0]]; - fn.Properties.SourceBucketNames = [{'Fn::Sub': '%%BUCKET_NAME%%-${AWS::Region}'}]; - } - else if (fn.Properties.hasOwnProperty('PolicyName') && fn.Properties.PolicyName.includes('CustomCDKBucketDeployment')) { - // Set CDKBucketDeployment S3 bucket Policy reference - fn.Properties.PolicyDocument.Statement.forEach(function(sub,i) { - if (typeof(sub.Resource[i]) === 'object') { - sub.Resource.forEach(function(resource){ - var arrayKey = Object.keys(resource); - if (typeof(resource[arrayKey][1]) === 'object') { - resource[arrayKey][1].filter(function(s){ - if (s.hasOwnProperty('Ref')) { - fn.Properties.PolicyDocument.Statement[i].Resource = [ - {"Fn::Join": ["",["arn:",{"Ref": "AWS::Partition"},":s3:::%%BUCKET_NAME%%-",{"Ref": "AWS::Region"}]]}, - {"Fn::Join": ["",["arn:",{"Ref": "AWS::Partition"},":s3:::%%BUCKET_NAME%%-",{"Ref": "AWS::Region"},"/*"]]}]}})}})}}); - } - // Set NestedStack S3 bucket reference - else if (fn.Properties.hasOwnProperty('TemplateURL')) { - var key=fn.Properties.TemplateURL['Fn::Join'][1][6].replace('.json','').replace('/',''); - var assetPath = assetRef(key); - fn.Properties.TemplateURL = { - "Fn::Join": ["", - [ - "https://s3.", - { - "Ref": "AWS::URLSuffix" - }, - "/", - `%%TEMPLATE_OUTPUT_BUCKET%%/%%SOLUTION_NAME%%/%%VERSION%%/${assetPath}` - ]] - }; - } - // Set CloudFront logging bucket - else if (fn.Properties.hasOwnProperty('DistributionConfig')){ - fn.Properties.DistributionConfig.Logging.Bucket= { - "Fn::Join": ["",[fn.Properties.DistributionConfig.Logging.Bucket['Fn::Join'][1][0], - ".s3.",{"Ref": "AWS::Region"},".",{"Ref": "AWS::URLSuffix"}]] - } - } - }); - - //6. Output modified template file - const output_template = JSON.stringify(template, null, 2); - fs.writeFileSync(`${global_s3_assets}/${file}`, output_template); - } -}); diff --git a/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/package.json b/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/package.json deleted file mode 100644 index cac79bed0..000000000 --- a/analytics/cdk/stream-emr-on-eks/deployment/cdk-solution-helper/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "cdk-solution-helper", - "version": "0.1.0", - "devDependencies": { - "fs": "0.0.1-security" - }, - "dependencies": { - "aws-cdk": "^2.67.0", - "fs": "0.0.1-security" - } -} diff --git a/analytics/cdk/stream-emr-on-eks/package.json b/analytics/cdk/stream-emr-on-eks/package.json deleted file mode 100644 index 5751c6f63..000000000 --- a/analytics/cdk/stream-emr-on-eks/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "aws-cdk": "^2.84.0" - } -} diff --git a/analytics/cdk/stream-emr-on-eks/requirements.txt b/analytics/cdk/stream-emr-on-eks/requirements.txt deleted file mode 100644 index f38fbf8b7..000000000 --- a/analytics/cdk/stream-emr-on-eks/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ --e . -# pytest diff --git a/analytics/cdk/stream-emr-on-eks/setup.py b/analytics/cdk/stream-emr-on-eks/setup.py deleted file mode 100644 index f25979b83..000000000 --- a/analytics/cdk/stream-emr-on-eks/setup.py +++ /dev/null @@ -1,52 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) -import setuptools - -try: - with open("README.md") as fp: - long_description = fp.read() -except IOError as e: - long_description = '' - -setuptools.setup( - name="emr-stream-demo", - version="2.0.0", - - description="A CDK Python app for Spark Structured Streaming ETL", - long_description=long_description, - long_description_content_type="text/markdown", - - author="meloyang", - - package_dir={"": "source"}, - packages=setuptools.find_packages(where="source"), - - install_requires=[ - "aws-cdk-lib==2.67.0", - "aws-cdk.aws-msk-alpha==2.67.0a0", - "aws-cdk.lambda-layer-kubectl-v24==2.0.118", - "constructs>=10.0.0,<11.0.0", - "pyyaml==5.4" - ], - - python_requires=">=3.6", - - classifiers=[ - "Development Status :: 4 - Beta", - - "Intended Audience :: Developers", - - "License :: OSI Approved :: MIT License", - - "Programming Language :: JavaScript", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - - "Topic :: Software Development :: Code Generators", - "Topic :: Utilities", - - "Typing :: Typed", - ], -) diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/00-deploy-to-aws.png b/analytics/cdk/stream-emr-on-eks/source/app_resources/00-deploy-to-aws.png deleted file mode 100644 index d6b7345f05a338fae5505c99a12354669bcff4f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 908 zcmV;719SX|P)`r5Ak`}O|%_4d4G^|)F2*{%Qn z{r~>``PHNO)1LRibN9h?{`Tzo(3kw>#`U*Y|NZ;?>(l$;z5C#l55I~d5c3~0w=Bp=c@ z{~neQsO{%$9fkdVa*QJa)aBa8V>8!a-QW0_Z0Q2l*hgkdM_6tW8GorN?bfUDLhQiu88S_CWiTld=Fp6lkQ6ymZf5MLpjJw~qLI=#O9zW&Q98r2 z31sQP*aD_)Y;n`?!V;)hlLC3l(L%q(Qm{grrXl}0)jE--sZaDWN##gY7@hb^SZ0?L zTuE^VYf~V&>G!A=Rz^_FK#C-UWl)7CsiqsZ5muIOykx^G>_JfpFB8-Fj+|c}1B+tI zsCuXYwY-EwI-r(R;f0jCXJAh`askngJ{cXn%aHt>)>!iS8BF zywzqkKPMLYo3Y1EOnpBm*5@7HMEpA2-s6UoGp4<3ZBLBlmXT3i)w^mJmtGI;e-OBj iXA8R?6VD$31^`48aKw7R6Z8N8002ovPDHLkV1f$%UhixG diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/alb-iam-role.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/alb-iam-role.yaml deleted file mode 100644 index d3c87663c..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/alb-iam-role.yaml +++ /dev/null @@ -1,166 +0,0 @@ -- Effect: Allow - Action: - - iam:CreateServiceLinkedRole - Resource: "*" - Condition: - StringEquals: - iam:AWSServiceName: elasticloadbalancing.amazonaws.com -- Effect: Allow - Action: - - ec2:DescribeAccountAttributes - - ec2:DescribeAddresses - - ec2:DescribeAvailabilityZones - - ec2:DescribeInternetGateways - - ec2:DescribeVpcs - - ec2:DescribeVpcPeeringConnections - - ec2:DescribeSubnets - - ec2:DescribeSecurityGroups - - ec2:DescribeInstances - - ec2:DescribeNetworkInterfaces - - ec2:DescribeTags - - ec2:GetCoipPoolUsage - - ec2:DescribeCoipPools - - elasticloadbalancing:DescribeLoadBalancers - - elasticloadbalancing:DescribeLoadBalancerAttributes - - elasticloadbalancing:DescribeListeners - - elasticloadbalancing:DescribeListenerCertificates - - elasticloadbalancing:DescribeSSLPolicies - - elasticloadbalancing:DescribeRules - - elasticloadbalancing:DescribeTargetGroups - - elasticloadbalancing:DescribeTargetGroupAttributes - - elasticloadbalancing:DescribeTargetHealth - - elasticloadbalancing:DescribeTags - Resource: "*" -- Effect: Allow - Action: - - cognito-idp:DescribeUserPoolClient - - acm:ListCertificates - - acm:DescribeCertificate - - iam:ListServerCertificates - - iam:GetServerCertificate - - waf-regional:GetWebACL - - waf-regional:GetWebACLForResource - - waf-regional:AssociateWebACL - - waf-regional:DisassociateWebACL - - wafv2:GetWebACL - - wafv2:GetWebACLForResource - - wafv2:AssociateWebACL - - wafv2:DisassociateWebACL - - shield:GetSubscriptionState - - shield:DescribeProtection - - shield:CreateProtection - - shield:DeleteProtection - Resource: "*" -- Effect: Allow - Action: - - ec2:AuthorizeSecurityGroupIngress - - ec2:RevokeSecurityGroupIngress - Resource: "*" -- Effect: Allow - Action: - - ec2:CreateSecurityGroup - Resource: "*" -- Effect: Allow - Action: - - ec2:CreateTags - Resource: arn:aws:ec2:*:*:security-group/* - Condition: - StringEquals: - ec2:CreateAction: CreateSecurityGroup - 'Null': - aws:RequestTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - ec2:CreateTags - - ec2:DeleteTags - Resource: arn:aws:ec2:*:*:security-group/* - Condition: - 'Null': - aws:RequestTag/elbv2.k8s.aws/cluster: 'true' - aws:ResourceTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - ec2:AuthorizeSecurityGroupIngress - - ec2:RevokeSecurityGroupIngress - - ec2:DeleteSecurityGroup - Resource: "*" - Condition: - 'Null': - aws:ResourceTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - elasticloadbalancing:CreateLoadBalancer - - elasticloadbalancing:CreateTargetGroup - Resource: "*" - Condition: - 'Null': - aws:RequestTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - elasticloadbalancing:CreateListener - - elasticloadbalancing:DeleteListener - - elasticloadbalancing:CreateRule - - elasticloadbalancing:DeleteRule - Resource: "*" -- Effect: Allow - Action: - - elasticloadbalancing:AddTags - - elasticloadbalancing:RemoveTags - Resource: - - arn:aws:elasticloadbalancing:*:*:targetgroup/*/* - - arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/* - - arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/* - Condition: - 'Null': - aws:RequestTag/elbv2.k8s.aws/cluster: 'true' - aws:ResourceTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - elasticloadbalancing:AddTags - - elasticloadbalancing:RemoveTags - Resource: - - arn:aws:elasticloadbalancing:*:*:listener/net/*/*/* - - arn:aws:elasticloadbalancing:*:*:listener/app/*/*/* - - arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/* - - arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/* -- Effect: Allow - Action: - - elasticloadbalancing:ModifyLoadBalancerAttributes - - elasticloadbalancing:SetIpAddressType - - elasticloadbalancing:SetSecurityGroups - - elasticloadbalancing:SetSubnets - - elasticloadbalancing:DeleteLoadBalancer - - elasticloadbalancing:ModifyTargetGroup - - elasticloadbalancing:ModifyTargetGroupAttributes - - elasticloadbalancing:DeleteTargetGroup - Resource: "*" - Condition: - 'Null': - aws:ResourceTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - elasticloadbalancing:AddTags - Resource: - - arn:aws:elasticloadbalancing:*:*:targetgroup/*/* - - arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/* - - arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/* - Condition: - StringEquals: - elasticloadbalancing:CreateAction: - - CreateTargetGroup - - CreateLoadBalancer - 'Null': - aws:RequestTag/elbv2.k8s.aws/cluster: 'false' -- Effect: Allow - Action: - - elasticloadbalancing:RegisterTargets - - elasticloadbalancing:DeregisterTargets - Resource: arn:aws:elasticloadbalancing:*:*:targetgroup/*/* -- Effect: Allow - Action: - - elasticloadbalancing:SetWebAcl - - elasticloadbalancing:ModifyListener - - elasticloadbalancing:AddListenerCertificates - - elasticloadbalancing:RemoveListenerCertificates - - elasticloadbalancing:ModifyRule - Resource: "*" diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/alb-values.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/alb-values.yaml deleted file mode 100644 index 531a78540..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/alb-values.yaml +++ /dev/null @@ -1,6 +0,0 @@ -region: {{region_name}} -vpcId: {{vpc_id}} -clusterName: {{cluster_name}} -serviceAccount: - create: false - name: alb-aws-load-balancer-controller diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-iam-role.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-iam-role.yaml deleted file mode 100644 index 4d6bba9b8..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-iam-role.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- Effect: Allow - Action: - - autoscaling:DescribeAutoScalingGroups - - autoscaling:DescribeAutoScalingInstances - - autoscaling:DescribeLaunchConfigurations - - autoscaling:DescribeTags - - autoscaling:SetDesiredCapacity - - autoscaling:TerminateInstanceInAutoScalingGroup - - ec2:DescribeLaunchTemplateVersions - Resource: - - "*" diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-values.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-values.yaml deleted file mode 100644 index aee016da9..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/autoscaler-values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -autoDiscovery: - clusterName: {{cluster_name}} -awsRegion: {{region_name}} -image: - tag: v1.19.1 -nodeSelector: - app: spark -podAnnotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: 'false' -extraArgs: - skip-nodes-with-system-pods: false - scale-down-unneeded-time: 5m - scale-down-unready-time: 10m -rbac: - serviceAccount: - create: false - name: cluster-autoscaler diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/emr-iam-role.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/emr-iam-role.yaml deleted file mode 100644 index 3f2e3c59d..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/emr-iam-role.yaml +++ /dev/null @@ -1,43 +0,0 @@ - - Effect: Allow - Action: - - s3:ListBucket - Resource: - - arn:aws:s3:::{{codeBucket}} - - arn:aws:s3:::amazon-reviews-pds - - Effect: Allow - Action: - - s3:PutObject - - s3:GetObject - Resource: - - arn:aws:s3:::{{codeBucket}}/* - - arn:aws:s3:::amazon-reviews-pds/parquet/* - - Effect: Allow - Action: - - s3:DeleteObject - Resource: - - arn:aws:s3:::{{codeBucket}}/* - - Effect: Allow - Action: - - kms:Encrypt - - kms:Decrypt - - kms:GenerateDataKey* - - kms:DescribeKey - Resource: - - '*' - - Effect: Allow - Action: - - logs:PutLogEvents - - logs:CreateLogStream - - logs:DescribeLogGroups - - logs:DescribeLogStreams - Resource: - - arn:aws:logs:*:*:* - - Effect: Allow - Action: - - kinesis:CreateStream - - kinesis:PutRecord - - kinesis:List* - - kinesis:Get* - - kinesis:DeleteStream - Resource: - - '*' diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/emr-rbac.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/emr-rbac.yaml deleted file mode 100644 index efd888912..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/emr-rbac.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: emr-containers - namespace: {{NAMESPACE}} -rules: - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts", "services", "configmaps", "events", "pods", "pods/log"] - verbs: ["get", "list", "watch", "describe", "create", "edit", "delete", "deletecollection", "annotate", "patch", "label"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "patch", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["statefulsets", "deployments"] - verbs: ["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"] - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"] - - apiGroups: ["extensions"] - resources: ["ingresses"] - verbs: ["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"] - - apiGroups: ["rbac.authorization.k8s.io"] - resources: ["roles", "rolebindings"] - verbs: ["get", "list", "watch", "describe", "create", "edit", "delete", "deletecollection", "annotate", "patch", "label"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: emr-containers - namespace: {{NAMESPACE}} -subjects: -- kind: User - name: emr-containers - apiGroup: rbac.authorization.k8s.io -roleRef: - kind: Role - name: emr-containers - apiGroup: rbac.authorization.k8s.io diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-iam-role.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-iam-role.yaml deleted file mode 100644 index 418b7bd0b..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-iam-role.yaml +++ /dev/null @@ -1,25 +0,0 @@ -- Effect: Allow - Action: s3:ListBucket - Resource: - - arn:aws:s3:::{{codeBucket}} - - arn:aws:s3:::amazon-reviews-pds -- Effect: Allow - Action: - - s3:PutObject - - s3:GetObject - Resource: - - arn:aws:s3:::{{codeBucket}}/* - - arn:aws:s3:::amazon-reviews-pds/* -- Effect: Allow - Action: - - s3:DeleteObject - Resource: - - arn:aws:s3:::{{codeBucket}}/* -- Effect: Allow - Action: - - kms:Encrypt - - kms:Decrypt - - kms:GenerateDataKey* - - kms:DescribeKey - Resource: - - '*' diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-rbac.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-rbac.yaml deleted file mode 100644 index d5130c763..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/native-spark-rbac.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{MY_SA}}-role-binding - namespace: spark -subjects: - - kind: ServiceAccount - name: {{MY_SA}} - namespace: spark -roleRef: - kind: Role - name: etl-workflow-role - apiGroup: rbac.authorization.k8s.io diff --git a/analytics/cdk/stream-emr-on-eks/source/app_resources/spark-operator-values.yaml b/analytics/cdk/stream-emr-on-eks/source/app_resources/spark-operator-values.yaml deleted file mode 100644 index 5e1fcbcc9..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/app_resources/spark-operator-values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -serviceAccounts: - spark: - create: false - sparkoperator: - create: true -metrics: -# -- Disable prometheus metric scraping - enable: false -webhook: - enable: true - port: 443 diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_base_app.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_base_app.py deleted file mode 100644 index 36625c443..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_base_app.py +++ /dev/null @@ -1,71 +0,0 @@ -###################################################################################################################### -# Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express o#implied. See the License for the specific language governing permissions # -# and limitations under the License. # # -###################################################################################################################### -from constructs import Construct -from aws_cdk import Aws -from aws_cdk.aws_eks import ICluster, KubernetesManifest -from lib.util.manifest_reader import * -import os - -class EksBaseAppConst(Construct): - def __init__(self,scope: Construct,id: str,eks_cluster: ICluster, **kwargs,) -> None: - super().__init__(scope, id, **kwargs) - - source_dir=os.path.split(os.environ['VIRTUAL_ENV'])[0]+'/source' - - # Add ALB ingress controller to EKS - self._alb = eks_cluster.add_helm_chart('ALBChart', - chart='aws-load-balancer-controller', - repository='https://aws.github.io/eks-charts', - release='alb', - version='1.4.8', - create_namespace=False, - namespace='kube-system', - values=load_yaml_replace_var_local(source_dir+'/app_resources/alb-values.yaml', - fields={ - "{{region_name}}": Aws.REGION, - "{{cluster_name}}": eks_cluster.cluster_name, - "{{vpc_id}}": eks_cluster.vpc.vpc_id - } - ) - ) - - # Add Cluster Autoscaler to EKS - _var_mapping = { - "{{region_name}}": Aws.REGION, - "{{cluster_name}}": eks_cluster.cluster_name, - } - eks_cluster.add_helm_chart('ClusterAutoScaler', - chart='cluster-autoscaler', - repository='https://kubernetes.github.io/autoscaler', - release='nodescaler', - create_namespace=False, - namespace='kube-system', - values=load_yaml_replace_var_local(source_dir+'/app_resources/autoscaler-values.yaml',_var_mapping) - ) - # Add container insight (CloudWatch Log) to EKS - KubernetesManifest(self,'ContainerInsight', - cluster=eks_cluster, - manifest=load_yaml_replace_var_remotely('https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml', - fields=_var_mapping, - multi_resource=True - ) - ) - # # Add Spark Operator to EKS - # eks_cluster.add_helm_chart('SparkOperatorChart', - # chart='spark-operator', - # repository='https://googlecloudplatform.github.io/spark-on-k8s-operator', - # release='spark-operator', - # version='1.1.27', - # create_namespace=True, - # values=load_yaml_replace_var_local(source_dir+'/app_resources/spark-operator-values.yaml',fields={'':''}) - # ) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_cluster.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_cluster.py deleted file mode 100644 index a2e1fe9f0..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_cluster.py +++ /dev/null @@ -1,83 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) - -from constructs import Construct -from aws_cdk import (aws_eks as eks, aws_ec2 as ec2) -from aws_cdk.aws_iam import IRole -from aws_cdk.lambda_layer_kubectl_v24 import KubectlV24Layer - -class EksConst(Construct): - - @property - def my_cluster(self): - return self._my_cluster - - @property - def awsAuth(self): - return self._my_cluster.aws_auth - - def __init__(self, scope: Construct, id:str, - eksname: str, - eksvpc: ec2.IVpc, - noderole: IRole, - eks_adminrole: IRole, - emr_svc_role: IRole, - fg_pod_role: IRole, - **kwargs - ) -> None: - super().__init__(scope, id, **kwargs) - - # 1.Create EKS cluster without node group - self._my_cluster = eks.Cluster(self,'EKS', - vpc= eksvpc, - cluster_name=eksname, - masters_role=eks_adminrole, - output_cluster_name=True, - version= eks.KubernetesVersion.V1_24, - endpoint_access= eks.EndpointAccess.PUBLIC_AND_PRIVATE, - default_capacity=0, - kubectl_layer=KubectlV24Layer(self, 'KubectlV24Layer') - ) - - - # 2.Add Managed NodeGroup to EKS, compute resource to run Spark jobs - self._my_cluster.add_nodegroup_capacity('onDemand-mn', - nodegroup_name = 'etl-ondemand', - node_role = noderole, - desired_size = 1, - max_size = 5, - disk_size = 50, - instance_types = [ec2.InstanceType('m5.xlarge')], - labels = {'app':'spark', 'lifecycle':'OnDemand'}, - subnets = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS,one_per_az=True), - tags = {'Name':'OnDemand-'+eksname,'k8s.io/cluster-autoscaler/enabled': 'true', 'k8s.io/cluster-autoscaler/'+eksname: 'owned'} - ) - - - # 3. Add Spot managed NodeGroup to EKS (Run Spark executor on spot) - self._my_cluster.add_nodegroup_capacity('spot-mn', - nodegroup_name = 'etl-spot', - node_role = noderole, - capacity_type=eks.CapacityType.SPOT, - desired_size = 1, - max_size = 30, - disk_size = 50, - instance_types=[ec2.InstanceType("r5.xlarge"),ec2.InstanceType("r4.xlarge"),ec2.InstanceType("r5a.xlarge")], - labels = {'app':'spark', 'lifecycle':'Ec2Spot'}, - tags = {'Name':'Spot-'+eksname, 'k8s.io/cluster-autoscaler/enabled': 'true', 'k8s.io/cluster-autoscaler/'+eksname: 'owned'} - ) - - # 4. Add Fargate NodeGroup to EKS, without setup cluster-autoscaler - self._my_cluster.add_fargate_profile('FargateEnabled', - selectors =[{ - "namespace": "emr", - "labels": {"type":"serverless"} - }], - pod_execution_role=fg_pod_role - ) - - # 5. Map EMR user to IAM role - self._my_cluster.aws_auth.add_role_mapping(emr_svc_role, groups=[], username="emr-containers") - - # # 6. Allow EKS access from Cloud9 - self._my_cluster.cluster_security_group.add_ingress_rule(ec2.Peer.ipv4(eksvpc.vpc_cidr_block),ec2.Port.all_tcp()) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_service_account.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_service_account.py deleted file mode 100644 index a316cde96..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/eks_service_account.py +++ /dev/null @@ -1,46 +0,0 @@ -###################################################################################################################### -# Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express o#implied. See the License for the specific language governing permissions # -# and limitations under the License. # # -###################################################################################################################### -from constructs import Construct -from aws_cdk import aws_iam as iam -from aws_cdk.aws_eks import ICluster -from lib.util.manifest_reader import * -import os - -class EksSAConst(Construct): - - def __init__(self, scope: Construct, id:str, eks_cluster: ICluster, **kwargs,) -> None: - super().__init__(scope, id, **kwargs) - - source_dir=os.path.split(os.environ['VIRTUAL_ENV'])[0]+'/source' - -# //************************************v*************************************************************// -# //***************************** SERVICE ACCOUNT, RBAC and IAM ROLES *******************************// -# //****** Associating IAM role to K8s Service Account to provide fine-grain security control ******// -# //***********************************************************************************************// - # Cluster Auto-scaler - self._scaler_sa = eks_cluster.add_service_account('AutoScalerSa', - name='cluster-autoscaler', - namespace='kube-system' - ) - _scaler_role = load_yaml_local(source_dir+'/app_resources/autoscaler-iam-role.yaml') - for statmt in _scaler_role: - self._scaler_sa.add_to_principal_policy(iam.PolicyStatement.from_json(statmt)) - - # ALB Ingress - self._alb_sa = eks_cluster.add_service_account('ALBServiceAcct', - name='alb-aws-load-balancer-controller', - namespace='kube-system' - ) - _alb_role = load_yaml_local(source_dir+'/app_resources/alb-iam-role.yaml') - for statmt in _alb_role: - self._alb_sa.add_to_principal_policy(iam.PolicyStatement.from_json(statmt)) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/iam_roles.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/iam_roles.py deleted file mode 100644 index c245ae62f..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/iam_roles.py +++ /dev/null @@ -1,115 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) - -from constructs import Construct -from aws_cdk import (RemovalPolicy, Tags, Aws, aws_iam as iam) -# import typing - -class IamConst(Construct): - - @property - def managed_node_role(self): - return self._managed_node_role - - @property - def admin_role(self): - return self._clusterAdminRole - - @property - def fg_pod_role(self): - return self._fg_pod_role - - @property - def emr_svc_role(self): - return self._emrsvcrole - - def __init__(self,scope: Construct, id:str, cluster_name:str, **kwargs,) -> None: - super().__init__(scope, id, **kwargs) - - # EKS admin role - self._clusterAdminRole = iam.Role(self, 'ClusterAdmin', - assumed_by= iam.AccountRootPrincipal() - ) - self._clusterAdminRole.add_to_policy(iam.PolicyStatement( - resources=["*"], - actions=[ - "eks:Describe*", - "eks:List*", - "eks:AccessKubernetesApi", - "ssm:GetParameter", - "iam:ListRoles" - ], - )) - Tags.of(self._clusterAdminRole).add( - key='eks/%s/type' % cluster_name, - value='admin-role' - ) - - # Managed Node Group Instance Role - _managed_node_managed_policies = ( - iam.ManagedPolicy.from_aws_managed_policy_name('AmazonEKSWorkerNodePolicy'), - iam.ManagedPolicy.from_aws_managed_policy_name('AmazonEKS_CNI_Policy'), - iam.ManagedPolicy.from_aws_managed_policy_name('AmazonEC2ContainerRegistryReadOnly'), - iam.ManagedPolicy.from_aws_managed_policy_name('CloudWatchAgentServerPolicy'), - ) - self._managed_node_role = iam.Role(self,'NodeInstanceRole', - path='/', - assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'), - managed_policies=list(_managed_node_managed_policies), - ) - self._managed_node_role.apply_removal_policy(RemovalPolicy.DESTROY) - - # Fargate pod execution role - self._fg_pod_role = iam.Role(self, "FargatePodExecRole", - path='/', - assumed_by=iam.ServicePrincipal('eks-fargate-pods.amazonaws.com'), - managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name('AmazonEKSFargatePodExecutionRolePolicy')] - ) - - # EMR container service role - self._emrsvcrole = iam.Role.from_role_arn(self, "EmrSvcRole", - role_arn=f"arn:aws:iam::{Aws.ACCOUNT_ID}:role/AWSServiceRoleForAmazonEMRContainers", - mutable=False - ) - - # Cloud9 EC2 role - self._cloud9_role=iam.Role(self,"Cloud9Admin", - path='/', - assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'), - managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name('AWSCloudFormationReadOnlyAccess')] - ) - self._cloud9_role.add_to_policy(iam.PolicyStatement( - resources=[self._clusterAdminRole.role_arn], - actions=["sts:AssumeRole"] - )) - self._cloud9_role.add_to_policy(iam.PolicyStatement( - resources=["*"], - actions=[ - "eks:Describe*", - "ssm:GetParameter", - "kafka:DescribeCluster", - "kafka:UpdateClusterConfiguration", - "s3:List*", - "s3:GetObject", - "elasticmapreduce:ListClusters", - "elasticmapreduce:DescribeCluster", - "elasticmapreduce:AddJobFlowSteps" - ] - )) - self._cloud9_role.add_to_policy(iam.PolicyStatement( - resources=[f"arn:aws:kafka:{Aws.REGION}:{Aws.ACCOUNT_ID}:/v1/clusters"], - actions=["kafka:ListClusters"] - )) - self._cloud9_role.add_to_policy(iam.PolicyStatement( - resources=[f"arn:aws:kafka:{Aws.REGION}:{Aws.ACCOUNT_ID}:/v1/configurations"], - actions=["kafka:CreateConfiguration","kafka:ListConfigurations"] - )) - self._cloud9_role.add_to_policy(iam.PolicyStatement( - resources=[f"arn:aws:emr-containers:{Aws.REGION}:{Aws.ACCOUNT_ID}:/virtualclusters/*"], - actions=["emr-containers:StartJobRun"] - )) - iam.CfnInstanceProfile(self,"Cloud9RoleProfile", - roles=[ self._cloud9_role.role_name], - instance_profile_name= self._cloud9_role.role_name - ) - self._cloud9_role.apply_removal_policy(RemovalPolicy.DESTROY) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/network_sg.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/network_sg.py deleted file mode 100644 index f817294a3..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/network_sg.py +++ /dev/null @@ -1,46 +0,0 @@ -###################################################################################################################### -# Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express o#implied. See the License for the specific language governing permissions # -# and limitations under the License. # # -###################################################################################################################### -from constructs import Construct -from aws_cdk import (Tags, aws_ec2 as ec2) - -class NetworkSgConst(Construct): - - @property - def vpc(self): - return self._vpc - - def __init__(self,scope: Construct, id:str, eksname:str, **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - # //*************************************************// - # //******************* NETWORK ********************// - # //************************************************// - # create VPC - self._vpc = ec2.Vpc(self, 'eksVpc',max_azs=2, nat_gateways=1) - Tags.of(self._vpc).add('Name', eksname + 'EksVpc') - - # VPC endpoint security group - self._vpc_endpoint_sg = ec2.SecurityGroup(self,'EndpointSg', - vpc=self._vpc, - description='Security Group for Endpoint', - ) - self._vpc_endpoint_sg.add_ingress_rule(ec2.Peer.ipv4(self._vpc.vpc_cidr_block),ec2.Port.tcp(port=443)) - Tags.of(self._vpc_endpoint_sg).add('Name','SparkOnEKS-VPCEndpointSg') - - # Add VPC endpoint - self._vpc.add_gateway_endpoint("S3GatewayEndpoint", - service=ec2.GatewayVpcEndpointAwsService.S3, - subnets=[ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), - ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)]) - - self._vpc.add_interface_endpoint("CWLogsEndpoint", service=ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS,security_groups=[self._vpc_endpoint_sg]) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/s3_app_code.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/s3_app_code.py deleted file mode 100644 index 895cedd77..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/s3_app_code.py +++ /dev/null @@ -1,31 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) -from constructs import Construct -from aws_cdk import (RemovalPolicy, aws_s3 as s3,aws_s3_deployment as s3deploy) -import os - -class S3AppCodeConst(Construct): - - @property - def code_bucket(self): - return self.bucket_name - - def __init__(self,scope: Construct, id: str, **kwargs,) -> None: - super().__init__(scope, id, **kwargs) - - # Upload application code to S3 bucket - self._artifact_bucket=s3.Bucket(self, id, - block_public_access=s3.BlockPublicAccess.BLOCK_ALL, - encryption=s3.BucketEncryption.KMS_MANAGED, - removal_policy= RemovalPolicy.DESTROY, - auto_delete_objects=True - ) - - proj_dir=os.path.split(os.environ['VIRTUAL_ENV'])[0] - self.deploy=s3deploy.BucketDeployment(self, "DeployCode", - sources=[s3deploy.Source.asset(proj_dir+'/deployment/app_code')], - destination_bucket= self._artifact_bucket, - destination_key_prefix="app_code", - memory_limit=256 - ) - self.bucket_name = self._artifact_bucket.bucket_name diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/spark_permission.py b/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/spark_permission.py deleted file mode 100644 index 6ec2b7ac8..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/cdk_infra/spark_permission.py +++ /dev/null @@ -1,207 +0,0 @@ -###################################################################################################################### -# Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express o#implied. See the License for the specific language governing permissions # -# and limitations under the License. # # -###################################################################################################################### -from constructs import Construct -from aws_cdk import (CfnJson, Aws, aws_iam as iam,aws_emrcontainers as emrc) -from aws_cdk.aws_eks import ICluster, KubernetesManifest, AwsAuth -from lib.util.manifest_reader import load_yaml_replace_var_local -import os - -class SparkOnEksConst(Construct): - - @property - def EMRVC(self): - return self.emr_vc.attr_id - - @property - def EMRFargateVC(self): - return self.emr_vc_fg.attr_id - - @property - def EMRExecRole(self): - return self._emr_exec_role.role_arn - - def __init__(self,scope: Construct, id: str, - eks_cluster: ICluster, - code_bucket: str, - awsAuth: AwsAuth, - **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - source_dir=os.path.split(os.environ['VIRTUAL_ENV'])[0]+'/source' - -# //****************************************************************************************// -# //************************** SETUP PERMISSION FOR OSS SPARK JOBS *************************// -# //******* create k8s namespace, service account, and IAM role for service account ********// -# //***************************************************************************************// - - # create k8s namespace - etl_ns = eks_cluster.add_manifest('SparkNamespace',{ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": "spark", - "labels": {"name":"spark"} - } - } - ) - - self._spark_sa = eks_cluster.add_service_account('NativeSparkSa', - name='nativejob', - namespace='spark' - ) - self._spark_sa.node.add_dependency(etl_ns) - - _spark_rb = eks_cluster.add_manifest('sparkRoleBinding', - load_yaml_replace_var_local(source_dir+'/app_resources/native-spark-rbac.yaml', - fields= { - "{{MY_SA}}": self._spark_sa.service_account_name - }) - ) - _spark_rb.node.add_dependency(self._spark_sa) - - _native_spark_iam = load_yaml_replace_var_local(source_dir+'/app_resources/native-spark-iam-role.yaml', - fields={ - "{{codeBucket}}": code_bucket - } - ) - for statmnt in _native_spark_iam: - self._spark_sa.add_to_principal_policy(iam.PolicyStatement.from_json(statmnt)) - -# # //*************************************************************************************// -# # //******************** SETUP PERMISSION FOR EMR ON EKS *****************************// -# # //***********************************************************************************// - - ################################# - ####### ####### - ####### EMR Namespace ####### - ####### ####### - ################################# - _emr_01_name = "emr" - emr_ns = eks_cluster.add_manifest('EMRNamespace',{ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": _emr_01_name, - "labels": {"name": _emr_01_name} - } - } - ) - _emr_02_name = "emrs-workshop" - emr_serverless_ns = eks_cluster.add_manifest('EMRFargateNamespace',{ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": _emr_02_name, - "labels": {"name": _emr_01_name} - } - } - ) - - ########################################### - ####### ####### - ####### k8s role for EMR on EKS ####### - ####### ####### - ########################################### - _emr_rb = KubernetesManifest(self,'EMRRoleBinding', - cluster=eks_cluster, - manifest=load_yaml_replace_var_local(source_dir+'/app_resources/emr-rbac.yaml', - fields= { - "{{NAMESPACE}}": _emr_01_name, - }, - multi_resource=True) - ) - _emr_rb.node.add_dependency(emr_ns) - - _emr_fg_rb = KubernetesManifest(self,'EMRFargateRoleBinding', - cluster=eks_cluster, - manifest=load_yaml_replace_var_local(source_dir+'/app_resources/emr-rbac.yaml', - fields= { - "{{NAMESPACE}}": _emr_02_name - }, - multi_resource=True) - ) - _emr_fg_rb.node.add_dependency(emr_serverless_ns) - - # Create EMR on EKS job executor role - ####################################### - ####### ####### - ####### EMR Execution Role ####### - ####### ####### - ####################################### - self._emr_exec_role = iam.Role(self, "EMRJobExecRole", assumed_by=iam.ServicePrincipal("eks.amazonaws.com")) - - # trust policy - _eks_oidc_provider=eks_cluster.open_id_connect_provider - _eks_oidc_issuer=_eks_oidc_provider.open_id_connect_provider_issuer - - sub_str_like = CfnJson(self, "ConditionJsonIssuer", - value={ - f"{_eks_oidc_issuer}:sub": f"system:serviceaccount:{_emr_01_name}:emr-containers-sa-*-*-{Aws.ACCOUNT_ID}-*" - } - ) - self._emr_exec_role.assume_role_policy.add_statements( - iam.PolicyStatement( - effect=iam.Effect.ALLOW, - actions=["sts:AssumeRoleWithWebIdentity"], - principals=[iam.OpenIdConnectPrincipal(_eks_oidc_provider, conditions={"StringLike": sub_str_like})]) - ) - - aud_str_like = CfnJson(self,"ConditionJsonAudEMR", - value={ - f"{_eks_oidc_issuer}:aud": "sts.amazon.com" - } - ) - self._emr_exec_role.assume_role_policy.add_statements( - iam.PolicyStatement( - effect=iam.Effect.ALLOW, - actions=["sts:AssumeRoleWithWebIdentity"], - principals=[iam.OpenIdConnectPrincipal(_eks_oidc_provider, conditions={"StringEquals": aud_str_like})] - ) - ) - # custom policy - _emr_iam = load_yaml_replace_var_local(source_dir+'/app_resources/emr-iam-role.yaml', - fields={ - "{{codeBucket}}": code_bucket - } - ) - for statmnt in _emr_iam: - self._emr_exec_role.add_to_policy(iam.PolicyStatement.from_json(statmnt)) - - - - ############################################ - ####### ####### - ####### EMR virtual Cluster Server ####### - ####### ####### - ############################################ - self.emr_vc = emrc.CfnVirtualCluster(self,"EMRCluster", - container_provider=emrc.CfnVirtualCluster.ContainerProviderProperty( - id=eks_cluster.cluster_name, - info=emrc.CfnVirtualCluster.ContainerInfoProperty(eks_info=emrc.CfnVirtualCluster.EksInfoProperty(namespace=_emr_01_name)), - type="EKS" - ), - name="EMROnEKS" - ) - self.emr_vc.node.add_dependency(self._emr_exec_role) - self.emr_vc.node.add_dependency(_emr_rb) - - self.emr_vc_fg = emrc.CfnVirtualCluster(self,"EMRServerlessCluster", - container_provider=emrc.CfnVirtualCluster.ContainerProviderProperty( - id=eks_cluster.cluster_name, - info=emrc.CfnVirtualCluster.ContainerInfoProperty(eks_info=emrc.CfnVirtualCluster.EksInfoProperty(namespace=_emr_02_name)), - type="EKS" - ), - name="EMROnEKSFargate" - ) - self.emr_vc_fg.node.add_dependency(self._emr_exec_role) - self.emr_vc_fg.node.add_dependency(_emr_fg_rb) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/emr_on_ec2_stack.py b/analytics/cdk/stream-emr-on-eks/source/lib/emr_on_ec2_stack.py deleted file mode 100644 index b3b85faff..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/emr_on_ec2_stack.py +++ /dev/null @@ -1,157 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) -# -from constructs import Construct -from aws_cdk import (Aws,NestedStack,RemovalPolicy,Tags,CfnTag,aws_iam as iam,aws_ec2 as ec2,aws_efs as efs) -from aws_cdk.aws_emr import CfnCluster -from lib.util.manifest_reader import load_yaml_replace_var_local -import os - -class EMREC2Stack(NestedStack): - - def __init__(self, scope: Construct, id: str, emr_version: str, cluster_name:str, eksvpc: ec2.IVpc, code_bucket:str, **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - source_dir=os.path.split(os.environ['VIRTUAL_ENV'])[0]+'/source' - # The VPC requires a Tag to allow EMR to create the relevant security groups - Tags.of(eksvpc).add("for-use-with-amazon-emr-managed-policies", "true") - - ####################################### - ####### ####### - ####### EFS for checkpointing ####### - ####### ####### - ####################################### - _efs_sg = ec2.SecurityGroup(self,'EFSSg', - security_group_name=cluster_name + '-EFS-sg', - vpc=eksvpc, - description='NFS access to EFS from EMR on EC2 cluster', - ) - _efs_sg.add_ingress_rule(ec2.Peer.ipv4(eksvpc.vpc_cidr_block),ec2.Port.tcp(port=2049)) - Tags.of(_efs_sg).add('Name', cluster_name+'-EFS-sg') - - _efs=efs.FileSystem(self,'EFSCheckpoint', - vpc=eksvpc, - security_group=_efs_sg, - encrypted=True, - lifecycle_policy=efs.LifecyclePolicy.AFTER_60_DAYS, - performance_mode=efs.PerformanceMode.MAX_IO, - removal_policy=RemovalPolicy.DESTROY, - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS, one_per_az=True) - ) - - ########################### - ####### ####### - ####### EMR Roles ####### - ####### ####### - ########################### - # emr job flow role - emr_job_role = iam.Role(self,"EMRJobRole", - assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), - managed_policies=[ - iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonElasticMapReduceforEC2Role"), - iam.ManagedPolicy.from_aws_managed_policy_name("AmazonMSKFullAccess"), - iam.ManagedPolicy.from_aws_managed_policy_name("AmazonElasticFileSystemReadOnlyAccess"), - iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore") - ] - ) - _iam = load_yaml_replace_var_local(source_dir+'/app_resources/emr-iam-role.yaml', - fields= { - "{{codeBucket}}": code_bucket - }) - for statmnt in _iam: - emr_job_role.add_to_policy(iam.PolicyStatement.from_json(statmnt) - ) - - # emr service role - svc_role = iam.Role(self,"EMRSVCRole", - assumed_by=iam.ServicePrincipal("elasticmapreduce.amazonaws.com"), - managed_policies=[ - iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEMRServicePolicy_v2") - ] - ) - svc_role.add_to_policy( - iam.PolicyStatement( - actions=["iam:PassRole"], - resources=[emr_job_role.role_arn], - conditions={"StringEquals": {"iam:PassedToService": "ec2.amazonaws.com"}}, - ) - ) - - # emr job flow profile - emr_job_flow_profile = iam.CfnInstanceProfile(self,"EMRJobflowProfile", - roles=[emr_job_role.role_name], - instance_profile_name=emr_job_role.role_name - ) - - #################################### - ####### ####### - ####### Create EMR Cluster ####### - ####### ####### - #################################### - emr_c = CfnCluster(self,"emr_ec2_cluster", - name=cluster_name, - applications=[CfnCluster.ApplicationProperty(name="Spark")], - log_uri=f"s3://{code_bucket}/elasticmapreduce/", - release_label=emr_version, - visible_to_all_users=True, - service_role=svc_role.role_name, - job_flow_role=emr_job_role.role_name, - tags=[CfnTag(key="project", value=cluster_name)], - instances=CfnCluster.JobFlowInstancesConfigProperty( - termination_protected=False, - master_instance_group=CfnCluster.InstanceGroupConfigProperty( - instance_count=1, - instance_type="r5.xlarge", - market="ON_DEMAND" - ), - core_instance_group=CfnCluster.InstanceGroupConfigProperty( - instance_count=1, - instance_type="r5.xlarge", - market="ON_DEMAND", - ebs_configuration=CfnCluster.EbsConfigurationProperty( - ebs_block_device_configs=[CfnCluster.EbsBlockDeviceConfigProperty( - volume_specification=CfnCluster.VolumeSpecificationProperty( - size_in_gb=100, - volume_type='gp2')) - ]) - ), - ec2_subnet_id=eksvpc.public_subnets[0].subnet_id - ), - configurations=[ - # use python3 for pyspark - CfnCluster.ConfigurationProperty( - classification="spark-env", - configurations=[ - CfnCluster.ConfigurationProperty( - classification="export", - configuration_properties={ - "PYSPARK_PYTHON": "/usr/bin/python3", - "PYSPARK_DRIVER_PYTHON": "/usr/bin/python3", - }, - ) - ], - ), - # dedicate cluster to single jobs - CfnCluster.ConfigurationProperty( - classification="spark", - configuration_properties={"maximizeResourceAllocation": "true"}, - ), - ], - managed_scaling_policy=CfnCluster.ManagedScalingPolicyProperty( - compute_limits=CfnCluster.ComputeLimitsProperty( - unit_type="Instances", - maximum_capacity_units=10, - minimum_capacity_units=1, - maximum_core_capacity_units=1, - maximum_on_demand_capacity_units=1 - ) - ), - bootstrap_actions=[CfnCluster.BootstrapActionConfigProperty( - name="mountEFS", - script_bootstrap_action=CfnCluster.ScriptBootstrapActionConfigProperty( - path=f"s3://{code_bucket}/app_code/job/emr-mount-efs.sh", - args=[_efs.file_system_id, Aws.REGION] - ) - )] - ) - emr_c.add_dependency(emr_job_flow_profile) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/msk_stack.py b/analytics/cdk/stream-emr-on-eks/source/lib/msk_stack.py deleted file mode 100644 index 302add032..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/msk_stack.py +++ /dev/null @@ -1,60 +0,0 @@ -###################################################################################################################### -# Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express o#implied. See the License for the specific language governing permissions # -# and limitations under the License. # # -###################################################################################################################### - -from aws_cdk import (RemovalPolicy,NestedStack,aws_cloud9 as cloud9,aws_ec2 as ec2, aws_msk_alpha as msk) -from constructs import Construct -class MSKStack(NestedStack): - - @property - def Cloud9URL(self): - return self._c9env.ref - - @property - def MSKBroker(self): - return self._msk_cluster.bootstrap_brokers - - - def __init__(self, scope: Construct, id: str, cluster_name:str, eksvpc: ec2.IVpc, **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - # launch Cloud9 as Kafka client - self._c9env = cloud9.CfnEnvironmentEC2(self, "KafkaClientEnv", - name= "kafka_client", - instance_type="t3.small", - subnet_id=eksvpc.public_subnets[0].subnet_id, - automatic_stop_time_minutes=60 - ) - self._c9env.apply_removal_policy(RemovalPolicy.DESTROY) - - # create MSK Cluster - self._msk_cluster = msk.Cluster(self, "EMR-EKS-stream", - cluster_name=cluster_name, - kafka_version=msk.KafkaVersion.V2_8_1, - vpc=eksvpc, - ebs_storage_info=msk.EbsStorageInfo(volume_size=500), - encryption_in_transit=msk.EncryptionInTransitConfig( - enable_in_cluster=True, - client_broker=msk.ClientBrokerEncryption.TLS_PLAINTEXT - ), - instance_type=ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.SMALL), - removal_policy=RemovalPolicy.DESTROY, - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC,one_per_az=True) - ) - - for subnet in eksvpc.public_subnets: - self._msk_cluster.connections.allow_from(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(2181), "Zookeeper Plaintext") - self._msk_cluster.connections.allow_from(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(2182), "Zookeeper TLS") - self._msk_cluster.connections.allow_from(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(9092), "Broker Plaintext") - self._msk_cluster.connections.allow_from(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.tcp(9094), "Zookeeper Plaintext") - for subnet in eksvpc.private_subnets: - self._msk_cluster.connections.allow_from(ec2.Peer.ipv4(subnet.ipv4_cidr_block), ec2.Port.all_traffic(), "All private traffic") diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/spark_on_eks_stack.py b/analytics/cdk/stream-emr-on-eks/source/lib/spark_on_eks_stack.py deleted file mode 100644 index d0bb633a5..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/spark_on_eks_stack.py +++ /dev/null @@ -1,52 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) - -from aws_cdk import Stack -from constructs import Construct -from lib.cdk_infra.network_sg import NetworkSgConst -from lib.cdk_infra.iam_roles import IamConst -from lib.cdk_infra.eks_cluster import EksConst -from lib.cdk_infra.eks_service_account import EksSAConst -from lib.cdk_infra.eks_base_app import EksBaseAppConst -from lib.cdk_infra.s3_app_code import S3AppCodeConst -from lib.cdk_infra.spark_permission import SparkOnEksConst -from lib.util.manifest_reader import * - -class SparkOnEksStack(Stack): - - @property - def code_bucket(self): - return self.app_s3.code_bucket - - @property - def eksvpc(self): - return self.network_sg.vpc - - - @property - def EMRVC(self): - return self.emr.EMRVC - - @property - def EMRFargateVC(self): - return self.emr.EMRFargateVC - - @property - def EMRExecRole(self): - return self.emr.EMRExecRole - - def __init__(self, scope: Construct, id: str, eksname: str, **kwargs) -> None: - super().__init__(scope, id, **kwargs) - - # 1. a new bucket to store application code - self.app_s3 = S3AppCodeConst(self,'appcode') - - # 2. EKS base infra - self.network_sg = NetworkSgConst(self,'network-sg', eksname) - iam = IamConst(self,'iam_roles', eksname) - self.eks_cluster = EksConst(self,'eks_cluster', eksname, self.network_sg.vpc, iam.managed_node_role, iam.admin_role, iam.emr_svc_role, iam.fg_pod_role) - EksSAConst(self, 'eks_service_account', self.eks_cluster.my_cluster) - EksBaseAppConst(self, 'eks_base_app', self.eks_cluster.my_cluster) - - # 3. Setup Spark environment, Register for EMR on EKS - self.emr = SparkOnEksConst(self,'spark_permission',self.eks_cluster.my_cluster, self.app_s3.code_bucket, self.eks_cluster.awsAuth) diff --git a/analytics/cdk/stream-emr-on-eks/source/lib/util/manifest_reader.py b/analytics/cdk/stream-emr-on-eks/source/lib/util/manifest_reader.py deleted file mode 100644 index 8406d1f91..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/lib/util/manifest_reader.py +++ /dev/null @@ -1,91 +0,0 @@ -# // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# // SPDX-License-Identifier: License :: OSI Approved :: MIT No Attribution License (MIT-0) -import yaml -import urllib.request as request -import os.path as path -import sys - -def load_yaml_remotely(url, multi_resource=False): - try: - file_to_parse = request.urlopen(url) - if multi_resource: - yaml_data = list(yaml.full_load_all(file_to_parse)) - else: - yaml_data = yaml.full_load(file_to_parse) - # print(yaml_data) - except: - print("Cannot read yaml config file {}, check formatting." - "".format(file_to_parse)) - sys.exit(1) - - return yaml_data - -def load_yaml_local(yaml_file, multi_resource=False): - - file_to_parse=path.join(path.dirname(__file__), yaml_file) - if not path.exists(file_to_parse): - print("The file {} does not exist" - "".format(file_to_parse)) - sys.exit(1) - - try: - with open(file_to_parse, 'r') as yaml_stream: - if multi_resource: - yaml_data = list(yaml.full_load_all(yaml_stream)) - else: - yaml_data = yaml.full_load(yaml_stream) - # print(yaml_data) - except: - print("Cannot read yaml config file {}, check formatting." - "".format(file_to_parse)) - sys.exit(1) - - return yaml_data - -def load_yaml_replace_var_remotely(url, fields, multi_resource=False): - try: - with request.urlopen(url) as f: - file_to_replace = f.read().decode('utf-8') - for searchwrd,replwrd in fields.items(): - file_to_replace = file_to_replace.replace(searchwrd, replwrd) - - if multi_resource: - yaml_data = list(yaml.full_load_all(file_to_replace)) - else: - yaml_data = yaml.full_load(file_to_replace) - # print(yaml_data) - except request.URLError as e: - print(e.reason) - sys.exit(1) - - return yaml_data - - -def load_yaml_replace_var_local(yaml_file, fields, multi_resource=False, write_output=False): - - file_to_replace=path.join(path.dirname(__file__), yaml_file) - if not path.exists(file_to_replace): - print("The file {} does not exist" - "".format(file_to_replace)) - sys.exit(1) - - try: - with open(file_to_replace, 'r') as f: - filedata = f.read() - - for searchwrd, replwrd in fields.items(): - filedata = filedata.replace(searchwrd, replwrd) - if multi_resource: - yaml_data = list(yaml.full_load_all(filedata)) - else: - yaml_data = yaml.full_load(filedata) - if write_output: - with open(file_to_replace, "w") as f: - yaml.dump(yaml_data, f, default_flow_style=False, allow_unicode = True, sort_keys=False) - - # print(yaml_data) - except request.URLError as e: - print(e.reason) - sys.exit(1) - - return yaml_data diff --git a/analytics/cdk/stream-emr-on-eks/source/package.json b/analytics/cdk/stream-emr-on-eks/source/package.json deleted file mode 100644 index c74d5ed7f..000000000 --- a/analytics/cdk/stream-emr-on-eks/source/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "aws-cdk": "^2.67.0" - } -} diff --git a/analytics/terraform/emr-eks-ack/README.md b/analytics/terraform/emr-eks-ack/README.md deleted file mode 100644 index 0f8dd58a9..000000000 --- a/analytics/terraform/emr-eks-ack/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# EMR on EKS with ACK Controller and Crossplane - -This pattern is used to deploy the EKS Cluster with EMR on EKS ACK Controllers and Crossplane. -Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/blueprints/amazon-emr-on-eks/emr-eks-ack) to deploy this pattern and run sample tests. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | -| [helm](#requirement\_helm) | >= 2.4 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.47 | -| [aws.ecr](#provider\_aws.ecr) | >= 4.47 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | -| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | -| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | -| [emr\_ack](#module\_emr\_ack) | ./modules/emr-ack | n/a | -| [emr\_containers](#module\_emr\_containers) | terraform-aws-modules/emr/aws//modules/virtual-cluster | ~> 1.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.27"` | no | -| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"emr-eks-ack"` | no | -| [private\_subnets](#input\_private\_subnets) | Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet | `list(string)` | 
[
  "10.1.0.0/17",
  "10.1.128.0/18"
]
| no | -| [public\_subnets](#input\_public\_subnets) | Public Subnets CIDRs. 62 IPs per Subnet | `list(string)` | 
[
  "10.1.255.128/26",
  "10.1.255.192/26"
]
| no | -| [region](#input\_region) | Region | `string` | `"us-west-2"` | no | -| [tags](#input\_tags) | Default tags | `map(string)` | `{}` | no | -| [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR | `string` | `"10.1.0.0/16"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster | -| [cluster\_name](#output\_cluster\_name) | The Amazon Resource Name (ARN) of the cluster | -| [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | -| [emr\_on\_eks](#output\_emr\_on\_eks) | EMR on EKS | -| [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | - diff --git a/analytics/terraform/emr-eks-ack/addons.tf b/analytics/terraform/emr-eks-ack/addons.tf deleted file mode 100644 index 5e34c9985..000000000 --- a/analytics/terraform/emr-eks-ack/addons.tf +++ /dev/null @@ -1,72 +0,0 @@ -#--------------------------------------------------------------- -# IRSA for EBS CSI Driver -#--------------------------------------------------------------- -module "ebs_csi_driver_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 5.20" - role_name_prefix = format("%s-%s-", local.name, "ebs-csi-driver") - attach_ebs_csi_policy = true - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] - } - } - tags = local.tags -} - -#--------------------------------------------------------------- -# EKS Blueprints Addons -#--------------------------------------------------------------- -module "eks_blueprints_addons" { - source = "aws-ia/eks-blueprints-addons/aws" - version = "~> 1.2" - - cluster_name = module.eks.cluster_name - cluster_endpoint = module.eks.cluster_endpoint - cluster_version = module.eks.cluster_version - oidc_provider_arn = module.eks.oidc_provider_arn - - #--------------------------------------- - # Amazon EKS Managed Add-ons - #--------------------------------------- - eks_addons = { - aws-ebs-csi-driver = { - service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn - } - coredns = { - preserve = true - } - vpc-cni = { - preserve = true - } - kube-proxy = { - preserve = true - } - } - - #--------------------------------------- - # Kubernetes Add-ons - #--------------------------------------- - enable_metrics_server = true - enable_cluster_autoscaler = true - enable_aws_cloudwatch_metrics = true - - #--------------------------------------- - # AWS for FluentBit - DaemonSet - #--------------------------------------- - enable_aws_for_fluentbit = true - aws_for_fluentbit_cw_log_group = { - use_name_prefix = false - name = "/${local.name}/aws-fluentbit-logs" # Add-on creates this log group - retention_in_days = 30 - } - aws_for_fluentbit = { - values = [templatefile("${path.module}/helm-values/aws-for-fluentbit-values.yaml", { - region = local.region, - cloudwatch_log_group = "/${local.name}/aws-fluentbit-logs" - })] - } - - tags = local.tags -} diff --git a/analytics/terraform/emr-eks-ack/cleanup.sh b/analytics/terraform/emr-eks-ack/cleanup.sh deleted file mode 100755 index 8f67e60e5..000000000 --- a/analytics/terraform/emr-eks-ack/cleanup.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail - -read -p "Enter the region: " region -export AWS_DEFAULT_REGION=$region - -targets=( - "module.emr_ack" - "module.emr_containers" - "module.eks_blueprints_addons" - "module.eks" -) - -#------------------------------------------- -# Helpful to delete the stuck in "Terminating" namespaces -# Rerun the cleanup.sh script to detect and delete the stuck resources -#------------------------------------------- -terminating_namespaces=$(kubectl get namespaces --field-selector status.phase=Terminating -o json | jq -r '.items[].metadata.name') - -# If there are no terminating namespaces, exit the script -if [[ -z $terminating_namespaces ]]; then - echo "No terminating namespaces found" -fi - -for ns in $terminating_namespaces; do - echo "Terminating namespace: $ns" - kubectl get namespace $ns -o json | sed 's/"kubernetes"//' | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f - -done - -#------------------------------------------- -# Terraform destroy per module target -#------------------------------------------- -for target in "${targets[@]}" -do - terraform destroy -target="$target" -auto-approve - destroy_output=$(terraform destroy -target="$target" -auto-approve 2>&1) - if [[ $? -eq 0 && $destroy_output == *"Destroy complete"* ]]; then - echo "SUCCESS: Terraform destroy of $target completed successfully" - else - echo "FAILED: Terraform destroy of $target failed" - exit 1 - fi -done - -terraform destroy -auto-approve -destroy_output=$(terraform destroy -auto-approve 2>&1) -if [[ $? -eq 0 && $destroy_output == *"Destroy complete"* ]]; then - echo "SUCCESS: Terraform destroy of all targets completed successfully" -else - echo "FAILED: Terraform destroy of all targets failed" - exit 1 -fi diff --git a/analytics/terraform/emr-eks-ack/emr-ack.tf b/analytics/terraform/emr-eks-ack/emr-ack.tf deleted file mode 100644 index 5066da98c..000000000 --- a/analytics/terraform/emr-eks-ack/emr-ack.tf +++ /dev/null @@ -1,8 +0,0 @@ -module "emr_ack" { - source = "./modules/emr-ack" - - eks_cluster_id = module.eks.cluster_name - eks_oidc_provider_arn = module.eks.oidc_provider_arn - ecr_public_repository_username = data.aws_ecrpublic_authorization_token.token.user_name - ecr_public_repository_password = data.aws_ecrpublic_authorization_token.token.password -} diff --git a/analytics/terraform/emr-eks-ack/emr-eks.tf b/analytics/terraform/emr-eks-ack/emr-eks.tf deleted file mode 100644 index 786d8cc9d..000000000 --- a/analytics/terraform/emr-eks-ack/emr-eks.tf +++ /dev/null @@ -1,19 +0,0 @@ -module "emr_containers" { - source = "terraform-aws-modules/emr/aws//modules/virtual-cluster" - version = "~> 1.0" - - for_each = toset(["data-team-a", "data-team-b"]) - - eks_cluster_id = module.eks.cluster_name - oidc_provider_arn = module.eks.oidc_provider_arn - - name = "${module.eks.cluster_name}-emr-${each.value}" - namespace = "emr-${each.value}" - - role_name = "${module.eks.cluster_name}-emr-${each.value}" - iam_role_use_name_prefix = false - iam_role_description = "EMR Execution Role for emr-${each.value}" - iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonS3FullAccess"] # Attach additional policies for execution IAM Role - - tags = merge(local.tags, { Name = "emr-${each.value}" }) -} diff --git a/analytics/terraform/emr-eks-ack/examples/emr-virtualcluster.yaml b/analytics/terraform/emr-eks-ack/examples/emr-virtualcluster.yaml deleted file mode 100644 index d46bde0f5..000000000 --- a/analytics/terraform/emr-eks-ack/examples/emr-virtualcluster.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: emrcontainers.services.k8s.aws/v1alpha1 -kind: VirtualCluster -metadata: - name: my-ack-vc -spec: - name: my-ack-vc - containerProvider: - id: emr-eks-ack # your eks cluster name - type_: EKS - info: - eksInfo: - namespace: emr-data-team-a diff --git a/analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.sh b/analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.sh deleted file mode 100644 index aa059b2cb..000000000 --- a/analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Prompt the user for the required values -read -p "Enter the CloudWatch log group name: " cloudwatch_log_group -read -p "Enter the EMR execution role ARN: " emr_execution_role_arn - -# Generate a random ID using the date and time -random_id=$(date +%s) - -# Replace the placeholders in the YAML file with the user-provided values -sed -i "s/\${CLOUDWATCH_LOG_GROUP}/$cloudwatch_log_group/g" sample-pyspark-job.yaml -sed -i "s/\${RANDOM_ID}/$random_id/g" my-jobrun.yaml -sed -i "s/\${EMR_EXECUTION_ROLE_ARN}/$emr_execution_role_arn/g" sample-pyspark-job.yaml - -# Apply the modified YAML file with kubectl -kubectl apply -f sample-pyspark-job.yaml diff --git a/analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.yaml b/analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.yaml deleted file mode 100644 index a54f530b1..000000000 --- a/analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: emrcontainers.services.k8s.aws/v1alpha1 -kind: JobRun -metadata: - name: my-ack-jobrun-${RANDOM_ID} -spec: - name: my-ack-jobrun-${RANDOM_ID} - virtualClusterRef: - from: - name: my-ack-vc - executionRoleARN: ${EMR_EXECUTION_ROLE_ARN} - releaseLabel: "emr-6.8.0-latest" - jobDriver: - sparkSubmitJobDriver: - entryPoint: "local:///usr/lib/spark/examples/src/main/python/pi.py" - entryPointArguments: - sparkSubmitParameters: "--conf spark.executor.instances=2 --conf spark.executor.memory=1G --conf spark.executor.cores=1 --conf spark.driver.cores=1" - configurationOverrides: | - ApplicationConfiguration: null - MonitoringConfiguration: - CloudWatchMonitoringConfiguration: - LogGroupName: ${CLOUDWATCH_LOG_GROUP} - LogStreamNamePrefix: pi-job diff --git a/analytics/terraform/emr-eks-ack/helm-values/aws-for-fluentbit-values.yaml b/analytics/terraform/emr-eks-ack/helm-values/aws-for-fluentbit-values.yaml deleted file mode 100755 index 0f05a308b..000000000 --- a/analytics/terraform/emr-eks-ack/helm-values/aws-for-fluentbit-values.yaml +++ /dev/null @@ -1,80 +0,0 @@ -global: - -#hostNetwork and dnsPolicy are critical for enabling large clusters to avoid making calls to API server -# see this link https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#optional-feature-using-kubelet-to-get-metadata -hostNetwork: true -dnsPolicy: ClusterFirstWithHostNet - -service: - parsersFiles: - - /fluent-bit/parsers/parsers.conf - extraParsers: | - [PARSER] - Name kubernetes - Format regex - Regex ^(?[^_]+)\.(?.+)\.(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?[a-z0-9]{64})-$ - -input: - name: "tail" - enabled: true - tag: "systempods....-" - path: "/var/log/containers/*.log" - db: "/var/log/flb_kube.db" - memBufLimit: 5MB - skipLongLines: "On" - refreshInterval: 10 - extraInputs: | - multiline.parser docker, cri - Tag_Regex (?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$ - - -# NOTE: extraFilters config for using Kubelet to get the Metadata instead of talking to API server for large clusters -filter: - name: "kubernetes" - match: "systempods.*" - kubeURL: "https://kubernetes.default.svc.cluster.local:443" - mergeLog: "On" - mergeLogKey: "log_processed" - keepLog: "On" - k8sLoggingParser: "On" - k8sLoggingExclude: "Off" - bufferSize: "0" - extraFilters: | - Kube_Tag_Prefix systempods. - Regex_Parser kubernetes - Labels On - Annotations Off - Use_Kubelet true - Kubelet_Port 10250 - Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token - -# CATION: Do not use `cloudwatch` plugin. This Golang Plugin is not recommended by AWS anymore instead use C plugin(`cloudWatchLogs`) for better performance. -# cloudWatch: -# enabled: false - -# This is a new high performance C Plugin for CloudWatchLogs. See docs here https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch -cloudWatchLogs: - enabled: true - match: "systempods.*" - region: ${region} - logGroupName: ${cloudwatch_log_group} - autoCreateGroup: false - extraOutputs: | - log_key log - -# Resource config for large clusters -resources: - limits: - cpu: 1000m - memory: 1500Mi - requests: - cpu: 500m - memory: 500Mi - -## Assign a PriorityClassName to pods if set -priorityClassName: system-node-critical - -# This toleration allows Daemonset pod to be scheduled on any node, regardless of their Taints. -tolerations: - - operator: Exists diff --git a/analytics/terraform/emr-eks-ack/install.sh b/analytics/terraform/emr-eks-ack/install.sh deleted file mode 100755 index 76c8ef3ba..000000000 --- a/analytics/terraform/emr-eks-ack/install.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -read -p "Enter the region: " region -export AWS_DEFAULT_REGION=$region - -echo "Initializing ..." -terraform init || echo "\"terraform init\" failed" - -# List of Terraform modules to apply in sequence -targets=( - "module.vpc" - "module.eks" - "module.eks_blueprints_addons" - "module.emr_containers" - "module.emr_ack" -) - -# Apply modules in sequence -for target in "${targets[@]}" -do - echo "Applying module $target..." - apply_output=$(terraform apply -target="$target" -var="region=$region" -auto-approve 2>&1 | tee /dev/tty) - if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then - echo "SUCCESS: Terraform apply of $target completed successfully" - else - echo "FAILED: Terraform apply of $target failed" - exit 1 - fi -done - -# Final apply to catch any remaining resources -echo "Applying remaining resources..." -apply_output=$(terraform apply -var="region=$region" -auto-approve 2>&1 | tee /dev/tty) -if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then - echo "SUCCESS: Terraform apply of all modules completed successfully" -else - echo "FAILED: Terraform apply of all modules failed" - exit 1 -fi diff --git a/analytics/terraform/emr-eks-ack/main.tf b/analytics/terraform/emr-eks-ack/main.tf deleted file mode 100644 index a5296b569..000000000 --- a/analytics/terraform/emr-eks-ack/main.tf +++ /dev/null @@ -1,148 +0,0 @@ -provider "aws" { - region = local.region -} - -# ECR always authenticates with `us-east-1` region -# Docs -> https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html -provider "aws" { - alias = "ecr" - region = "us-east-1" -} - -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.this.token -} - -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.this.token - } -} - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_name -} - -data "aws_ecrpublic_authorization_token" "token" { - provider = aws.ecr -} - -data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} - -locals { - name = var.name - region = var.region - - vpc_cidr = var.vpc_cidr - azs = slice(data.aws_availability_zones.available.names, 0, 2) - - tags = merge(var.tags, { - Blueprint = local.name - GithubRepo = "github.com/awslabs/data-on-eks" - }) -} - -#--------------------------------------------------------------- -# EKS Cluster -#--------------------------------------------------------------- - -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.15" - - cluster_name = local.name - cluster_version = var.eks_cluster_version - - cluster_endpoint_private_access = true # if true, Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint - cluster_endpoint_public_access = true # if true, Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - manage_aws_auth_configmap = true - aws_auth_roles = [ - { - # Required for EMR on EKS virtual cluster - rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/AWSServiceRoleForAmazonEMRContainers" - username = "emr-containers" - }, - ] - - #--------------------------------------- - # Note: This can further restricted to specific required for each Add-on and your application - #--------------------------------------- - # Extend cluster security group rules - cluster_security_group_additional_rules = { - ingress_nodes_ephemeral_ports_tcp = { - description = "Nodes on ephemeral ports" - protocol = "tcp" - from_port = 1025 - to_port = 65535 - type = "ingress" - source_node_security_group = true - } - } - - # Extend node-to-node security group rules - node_security_group_additional_rules = { - # Extend node-to-node security group rules. Recommended and required for the Add-ons - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - } - - eks_managed_node_group_defaults = { - iam_role_additional_policies = { - # Not required, but used in the example to access the nodes to inspect mounted volumes - AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - } - } - eks_managed_node_groups = { - # We recommend to have a MNG to place your critical workloads and add-ons - # Then rely on Karpenter to scale your workloads - # You can also make uses on nodeSelector and Taints/tolerations to spread workloads on MNG or Karpenter provisioners - core_node_group = { - name = "core-node-group" - description = "EKS managed node group example launch template" - - min_size = 1 - max_size = 9 - desired_size = 3 - - instance_types = ["m5.xlarge"] - - ebs_optimized = true - block_device_mappings = { - xvda = { - device_name = "/dev/xvda" - ebs = { - volume_size = 100 - volume_type = "gp3" - } - } - } - - labels = { - WorkerType = "ON_DEMAND" - NodeGroupType = "core" - } - - tags = { - Name = "core-node-grp", - "karpenter.sh/discovery" = local.name - } - } - } - - tags = local.tags -} diff --git a/analytics/terraform/emr-eks-ack/modules/emr-ack/README.md b/analytics/terraform/emr-eks-ack/modules/emr-ack/README.md deleted file mode 100644 index b876ef828..000000000 --- a/analytics/terraform/emr-eks-ack/modules/emr-ack/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# EMR ACK controllers Terraform Module - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.13 | -| [helm](#requirement\_helm) | >= 2.4.1 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.13 | -| [helm](#provider\_helm) | >= 2.4.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [emr\_containers\_irsa](#module\_emr\_containers\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.14 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.emr_containers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [helm_release.emr_containers](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ecr\_public\_repository\_password](#input\_ecr\_public\_repository\_password) | ECR Public repository Password for Helm Charts | `string` | n/a | yes | -| [ecr\_public\_repository\_username](#input\_ecr\_public\_repository\_username) | ECR Public repository Username for Helm Charts | `string` | n/a | yes | -| [eks\_cluster\_id](#input\_eks\_cluster\_id) | Name of the EKS Cluster | `string` | n/a | yes | -| [eks\_oidc\_provider\_arn](#input\_eks\_oidc\_provider\_arn) | The OpenID Connect identity provider ARN | `string` | n/a | yes | -| [helm\_config](#input\_helm\_config) | EMR ACK Controller Helm Chart values | `any` | `{}` | no | -| [tags](#input\_tags) | Tags to apply to all AWS resources | `map(string)` | `{}` | no | - -## Outputs - -No outputs. - diff --git a/analytics/terraform/emr-eks-ack/modules/emr-ack/main.tf b/analytics/terraform/emr-eks-ack/modules/emr-ack/main.tf deleted file mode 100644 index 1907d8252..000000000 --- a/analytics/terraform/emr-eks-ack/modules/emr-ack/main.tf +++ /dev/null @@ -1,165 +0,0 @@ -data "aws_region" "current" {} - -locals { - service = "emrcontainers" - name = "ack-emrcontainers-controller" - region = data.aws_region.current.name - - set_values = [ - { - name = "serviceAccount.name" - value = local.name - }, - { - name = "aws.region" - value = local.region - }, - { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.emr_containers_irsa.iam_role_arn - }, - ] -} - -#--------------------------------------------------------------- -# EMR on EKS ACK Addon -#--------------------------------------------------------------- -resource "helm_release" "emr_containers" { - name = try(var.helm_config["name"], local.name) - repository = try(var.helm_config["repository"], "oci://public.ecr.aws/aws-controllers-k8s") - chart = try(var.helm_config["chart"], "${local.service}-chart") - version = try(var.helm_config["version"], "v1.0.0") - namespace = try(var.helm_config["namespace"], local.name) - description = try(var.helm_config["description"], "Helm Charts for the emr-containers controller for AWS Controllers for Kubernetes (ACK)") - create_namespace = try(var.helm_config["create_namespace"], true) - repository_username = var.ecr_public_repository_username - repository_password = var.ecr_public_repository_password - timeout = try(var.helm_config["timeout"], "300") - - values = try(var.helm_config["values"], []) - - dynamic "set" { - iterator = each_item - for_each = try(var.helm_config["set"], null) != null ? distinct(concat(local.set_values, var.helm_config["set"])) : local.set_values - - content { - name = each_item.value.name - value = each_item.value.value - type = try(each_item.value.type, null) - } - } - - dynamic "set_sensitive" { - iterator = each_item - for_each = try(var.helm_config["set_sensitive"], []) - - content { - name = each_item.value.name - value = each_item.value.value - type = try(each_item.value.type, null) - } - } -} - -#--------------------------------------------------------------- -# IRSA IAM policy for EMR Containers -#--------------------------------------------------------------- -resource "aws_iam_policy" "emr_containers" { - name_prefix = format("%s-%s", var.eks_cluster_id, "emr-ack") - description = "IAM policy for EMR Containers controller" - path = "/" - policy = data.aws_iam_policy_document.emrcontainers.json -} - -#--------------------------------------------------------------- -# IRSA for EMR containers -#--------------------------------------------------------------- -module "emr_containers_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 5.14" - role_name = format("%s-%s", var.eks_cluster_id, local.name) - - role_policy_arns = { - policy = aws_iam_policy.emr_containers.arn - } - oidc_providers = { - main = { - provider_arn = var.eks_oidc_provider_arn - namespace_service_accounts = ["${local.name}:${local.name}"] - } - } - tags = var.tags -} - -# inline policy providered by ack https://raw.githubusercontent.com/aws-controllers-k8s/emrcontainers-controller/main/config/iam/recommended-inline-policy -data "aws_iam_policy_document" "emrcontainers" { - statement { - effect = "Allow" - actions = [ - "iam:CreateServiceLinkedRole" - ] - resources = ["*"] - - condition { - test = "StringLike" - variable = "iam:AWSServiceName" - values = ["emr-containers.amazonaws.com"] - } - } - - statement { - effect = "Allow" - actions = [ - "emr-containers:CreateVirtualCluster", - "emr-containers:ListVirtualClusters", - "emr-containers:DescribeVirtualCluster", - "emr-containers:DeleteVirtualCluster" - ] - resources = ["*"] - } - - statement { - effect = "Allow" - actions = [ - "emr-containers:StartJobRun", - "emr-containers:ListJobRuns", - "emr-containers:DescribeJobRun", - "emr-containers:CancelJobRun" - ] - - resources = ["*"] - } - - statement { - effect = "Allow" - actions = [ - "emr-containers:DescribeJobRun", - "emr-containers:TagResource", - "elasticmapreduce:CreatePersistentAppUI", - "elasticmapreduce:DescribePersistentAppUI", - "elasticmapreduce:GetPersistentAppUIPresignedURL" - ] - - resources = ["*"] - } - - statement { - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket" - ] - - resources = ["*"] - } - - statement { - effect = "Allow" - actions = [ - "logs:Get*", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams" - ] - resources = ["*"] - } -} diff --git a/analytics/terraform/emr-eks-ack/modules/emr-ack/outputs.tf b/analytics/terraform/emr-eks-ack/modules/emr-ack/outputs.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/analytics/terraform/emr-eks-ack/modules/emr-ack/variables.tf b/analytics/terraform/emr-eks-ack/modules/emr-ack/variables.tf deleted file mode 100644 index ec701c788..000000000 --- a/analytics/terraform/emr-eks-ack/modules/emr-ack/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -variable "helm_config" { - description = "EMR ACK Controller Helm Chart values" - type = any - default = {} -} - -variable "eks_cluster_id" { - description = "Name of the EKS Cluster" - type = string -} - -variable "eks_oidc_provider_arn" { - description = "The OpenID Connect identity provider ARN" - type = string -} - -variable "ecr_public_repository_username" { - description = "ECR Public repository Username for Helm Charts" - type = string -} - -variable "ecr_public_repository_password" { - description = "ECR Public repository Password for Helm Charts" - type = string -} - -variable "tags" { - description = "Tags to apply to all AWS resources" - type = map(string) - default = {} -} diff --git a/analytics/terraform/emr-eks-ack/modules/emr-ack/versions.tf b/analytics/terraform/emr-eks-ack/modules/emr-ack/versions.tf deleted file mode 100644 index 2f7a938c3..000000000 --- a/analytics/terraform/emr-eks-ack/modules/emr-ack/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.13" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4.1" - } - } -} diff --git a/analytics/terraform/emr-eks-ack/outputs.tf b/analytics/terraform/emr-eks-ack/outputs.tf deleted file mode 100644 index 66f966a78..000000000 --- a/analytics/terraform/emr-eks-ack/outputs.tf +++ /dev/null @@ -1,32 +0,0 @@ -################################################################################ -# Cluster -################################################################################ - -output "cluster_arn" { - description = "The Amazon Resource Name (ARN) of the cluster" - value = module.eks.cluster_arn -} - -output "cluster_name" { - description = "The Amazon Resource Name (ARN) of the cluster" - value = module.eks.cluster_id -} - -output "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`" - value = module.eks.oidc_provider_arn -} - -################################################################################ -# EKS Managed Node Group -################################################################################ - -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" -} - -output "emr_on_eks" { - description = "EMR on EKS" - value = module.emr_containers -} diff --git a/analytics/terraform/emr-eks-ack/variables.tf b/analytics/terraform/emr-eks-ack/variables.tf deleted file mode 100644 index e982fe380..000000000 --- a/analytics/terraform/emr-eks-ack/variables.tf +++ /dev/null @@ -1,42 +0,0 @@ -variable "name" { - description = "Name of the VPC and EKS Cluster" - type = string - default = "emr-eks-ack" -} - -variable "region" { - description = "Region" - type = string - default = "us-west-2" -} - -variable "eks_cluster_version" { - description = "EKS Cluster version" - type = string - default = "1.27" -} - -variable "tags" { - description = "Default tags" - type = map(string) - default = {} -} - -variable "vpc_cidr" { - description = "VPC CIDR" - type = string - default = "10.1.0.0/16" -} - -# Only two Subnets for with low IP range for internet access -variable "public_subnets" { - description = "Public Subnets CIDRs. 62 IPs per Subnet" - type = list(string) - default = ["10.1.255.128/26", "10.1.255.192/26"] -} - -variable "private_subnets" { - description = "Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet" - type = list(string) - default = ["10.1.0.0/17", "10.1.128.0/18"] -} diff --git a/analytics/terraform/emr-eks-ack/versions.tf b/analytics/terraform/emr-eks-ack/versions.tf deleted file mode 100644 index cecdcc00c..000000000 --- a/analytics/terraform/emr-eks-ack/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.47" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4" - } - } - - # ## Used for end-to-end testing on project; update to suit your needs - # backend "s3" { - # bucket = "doeks-github-actions-e2e-test-state" - # region = "us-west-2" - # key = "e2e/emr-eks-ack-crossplane/terraform.tfstate" - # } -} diff --git a/analytics/terraform/emr-eks-ack/vpc.tf b/analytics/terraform/emr-eks-ack/vpc.tf deleted file mode 100644 index ab607bb1b..000000000 --- a/analytics/terraform/emr-eks-ack/vpc.tf +++ /dev/null @@ -1,40 +0,0 @@ -#--------------------------------------------------------------- -# VPC and Subnets -#--------------------------------------------------------------- -# WARNING: This VPC module includes the creation of an Internet Gateway and NAT Gateway, which simplifies cluster deployment and testing, primarily intended for sandbox accounts. -# IMPORTANT: For preprod and prod use cases, it is crucial to consult with your security team and AWS architects to design a private infrastructure solution that aligns with your security requirements - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = local.name - cidr = local.vpc_cidr - azs = local.azs - - public_subnets = var.public_subnets - private_subnets = var.private_subnets - - enable_nat_gateway = true - single_nat_gateway = true - - # Manage so we can name - manage_default_network_acl = true - default_network_acl_tags = { Name = "${local.name}-default" } - manage_default_route_table = true - default_route_table_tags = { Name = "${local.name}-default" } - manage_default_security_group = true - default_security_group_tags = { Name = "${local.name}-default" } - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} diff --git a/analytics/terraform/emr-eks-fargate/README.md b/analytics/terraform/emr-eks-fargate/README.md deleted file mode 100644 index d0dbbbf4d..000000000 --- a/analytics/terraform/emr-eks-fargate/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# EMR on EKS with Apache YuniKorn Batch Scheduler - -Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/blueprints/amazon-emr-on-eks/emr-eks-fargate) to deploy this pattern and run sample tests. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | -| [helm](#requirement\_helm) | >= 2.4 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.47 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | -| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | -| [emr\_containers](#module\_emr\_containers) | terraform-aws-modules/emr/aws//modules/virtual-cluster | ~> 1.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.27"` | no | -| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"emr-eks-fargate"` | no | -| [private\_subnets](#input\_private\_subnets) | Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet | `list(string)` | 
[
  "10.1.0.0/17",
  "10.1.128.0/18"
]
| no | -| [public\_subnets](#input\_public\_subnets) | Public Subnets CIDRs. 62 IPs per Subnet | `list(string)` | 
[
  "10.1.255.128/26",
  "10.1.255.192/26"
]
| no | -| [region](#input\_region) | Region | `string` | `"us-west-2"` | no | -| [tags](#input\_tags) | Default tags | `map(string)` | `{}` | no | -| [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR | `string` | `"10.1.0.0/16"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | -| [emr\_on\_eks](#output\_emr\_on\_eks) | EMR on EKS | - diff --git a/analytics/terraform/emr-eks-fargate/addons.tf b/analytics/terraform/emr-eks-fargate/addons.tf deleted file mode 100644 index 9c6f22f4b..000000000 --- a/analytics/terraform/emr-eks-fargate/addons.tf +++ /dev/null @@ -1,58 +0,0 @@ -#--------------------------------------------------------------- -# EKS Blueprints Addons -#--------------------------------------------------------------- -module "eks_blueprints_addons" { - source = "aws-ia/eks-blueprints-addons/aws" - version = "~> 1.2" - - cluster_name = module.eks.cluster_name - cluster_endpoint = module.eks.cluster_endpoint - cluster_version = module.eks.cluster_version - oidc_provider_arn = module.eks.oidc_provider_arn - - #--------------------------------------- - # Amazon EKS Managed Add-ons - #--------------------------------------- - eks_addons = { - coredns = { - preserve = true - configuration_values = jsonencode({ - computeType = "Fargate" - # Ensure that the we fully utilize the minimum amount of resources that are supplied by - # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html - # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes - # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following - # compute configuration that most closely matches the sum of vCPU and memory requests in - # order to ensure pods always have the resources that they need to run. - resources = { - limits = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - requests = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - } - }) - } - vpc-cni = { - preserve = true - } - kube-proxy = { - preserve = true - } - } - - #--------------------------------------- - # Kubernetes Add-ons - #--------------------------------------- - - enable_fargate_fluentbit = true - - tags = local.tags -} diff --git a/analytics/terraform/emr-eks-fargate/cleanup.sh b/analytics/terraform/emr-eks-fargate/cleanup.sh deleted file mode 100755 index 54567063c..000000000 --- a/analytics/terraform/emr-eks-fargate/cleanup.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail - -read -p "Enter the region: " region -export AWS_DEFAULT_REGION=$region - -targets=( - "module.emr_containers" - "module.eks_blueprints_addons" - "module.eks" -) - -#------------------------------------------- -# Helpful to delete the stuck in "Terminating" namespaces -# Rerun the cleanup.sh script to detect and delete the stuck resources -#------------------------------------------- -terminating_namespaces=$(kubectl get namespaces --field-selector status.phase=Terminating -o json | jq -r '.items[].metadata.name') - -# If there are no terminating namespaces, exit the script -if [[ -z $terminating_namespaces ]]; then - echo "No terminating namespaces found" -fi - -for ns in $terminating_namespaces; do - echo "Terminating namespace: $ns" - kubectl get namespace $ns -o json | sed 's/"kubernetes"//' | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f - -done - -#------------------------------------------- -# Terraform destroy per module target -#------------------------------------------- -for target in "${targets[@]}" -do - terraform destroy -target="$target" -auto-approve - destroy_output=$(terraform destroy -target="$target" -auto-approve 2>&1) - if [[ $? -eq 0 && $destroy_output == *"Destroy complete!"* ]]; then - echo "SUCCESS: Terraform destroy of $target completed successfully" - else - echo "FAILED: Terraform destroy of $target failed" - exit 1 - fi -done - -terraform destroy -auto-approve -destroy_output=$(terraform destroy -auto-approve 2>&1) -if [[ $? -eq 0 && $destroy_output == *"Destroy complete!"* ]]; then - echo "SUCCESS: Terraform destroy of all targets completed successfully" -else - echo "FAILED: Terraform destroy of all targets failed" - exit 1 -fi diff --git a/analytics/terraform/emr-eks-fargate/emr-eks.tf b/analytics/terraform/emr-eks-fargate/emr-eks.tf deleted file mode 100644 index 786d8cc9d..000000000 --- a/analytics/terraform/emr-eks-fargate/emr-eks.tf +++ /dev/null @@ -1,19 +0,0 @@ -module "emr_containers" { - source = "terraform-aws-modules/emr/aws//modules/virtual-cluster" - version = "~> 1.0" - - for_each = toset(["data-team-a", "data-team-b"]) - - eks_cluster_id = module.eks.cluster_name - oidc_provider_arn = module.eks.oidc_provider_arn - - name = "${module.eks.cluster_name}-emr-${each.value}" - namespace = "emr-${each.value}" - - role_name = "${module.eks.cluster_name}-emr-${each.value}" - iam_role_use_name_prefix = false - iam_role_description = "EMR Execution Role for emr-${each.value}" - iam_role_additional_policies = ["arn:aws:iam::aws:policy/AmazonS3FullAccess"] # Attach additional policies for execution IAM Role - - tags = merge(local.tags, { Name = "emr-${each.value}" }) -} diff --git a/analytics/terraform/emr-eks-fargate/examples/basic-pyspark-job.sh b/analytics/terraform/emr-eks-fargate/examples/basic-pyspark-job.sh deleted file mode 100755 index 0d9bfeb03..000000000 --- a/analytics/terraform/emr-eks-fargate/examples/basic-pyspark-job.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ $# -eq 0 ]; -then - echo "$0: Missing arguments ENTER_EMR_EMR_VIRTUAL_CLUSTER_ID and EMR_JOB_EXECUTION_ROLE_ARN" - echo "USAGE: ./basic-pyspark-job '' ''" - exit 1 -elif [ $# -gt 3 ]; -then - echo "$0: Too many arguments: $@" - echo "Usage example-> ./basic-pyspark-job '' ''" - exit 1 -else - echo "We got some argument(s)" - echo "===========================" - echo "Number of arguments.: $#" - echo "List of arguments...: $@" - echo "Arg #1..............: $1" - echo "Arg #2..............: $2" - echo "===========================" -fi - -#-------------------------------------------- -# INPUT VARIABLES -#-------------------------------------------- -EMR_EMR_VIRTUAL_CLUSTER_ID=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` -EMR_JOB_EXECUTION_ROLE_ARN=$2 # Terraform output variable is emr_on_eks_role_arn -JOB_NAME='pi' -EMR_EKS_RELEASE_LABEL='emr-6.8.0-latest' - -#-------------------------------------------- -# DERIVED VARIABLES -#-------------------------------------------- -EMR_VIRTUAL_CLUSTER_NAME=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].name" --output text) - -# Execute Spark job -if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then - echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." - aws emr-containers start-job-run \ - --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ - --name $JOB_NAME \ - --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ - --release-label $EMR_EKS_RELEASE_LABEL \ - --job-driver '{ - "sparkSubmitJobDriver": { - "entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py", - "sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1" - } - }' - -else - echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" -fi diff --git a/analytics/terraform/emr-eks-fargate/install.sh b/analytics/terraform/emr-eks-fargate/install.sh deleted file mode 100755 index 069187bd9..000000000 --- a/analytics/terraform/emr-eks-fargate/install.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -read -p "Enter the region: " region -export AWS_DEFAULT_REGION=$region - -# List of Terraform modules to apply in sequence -targets=( - "module.vpc" - "module.eks" - "module.eks_blueprints_addons" - "module.emr_containers" -) - -# Apply modules in sequence -for target in "${targets[@]}" -do - echo "Applying module $target..." - apply_output=$(terraform apply -target="$target" -var="region=$region" -auto-approve 2>&1 | tee /dev/tty) - if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then - echo "SUCCESS: Terraform apply of $target completed successfully" - else - echo "FAILED: Terraform apply of $target failed" - exit 1 - fi -done - -# Final apply to catch any remaining resources -echo "Applying remaining resources..." -apply_output=$(terraform apply -var="region=$region" -auto-approve 2>&1 | tee /dev/tty) -if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then - echo "SUCCESS: Terraform apply of all modules completed successfully" -else - echo "FAILED: Terraform apply of all modules failed" - exit 1 -fi diff --git a/analytics/terraform/emr-eks-fargate/main.tf b/analytics/terraform/emr-eks-fargate/main.tf deleted file mode 100644 index c0282389c..000000000 --- a/analytics/terraform/emr-eks-fargate/main.tf +++ /dev/null @@ -1,82 +0,0 @@ -provider "aws" { - region = local.region -} - -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.this.token -} - -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.this.token - } -} - -data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_name -} -data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} - -locals { - name = var.name - region = var.region - - vpc_cidr = var.vpc_cidr - azs = slice(data.aws_availability_zones.available.names, 0, 2) - - tags = merge(var.tags, { - Blueprint = local.name - GithubRepo = "github.com/awslabs/data-on-eks" - }) -} - -#------------------------------------------------------------------ -# EKS Cluster -#------------------------------------------------------------------ - -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.15" - - cluster_name = local.name - cluster_version = var.eks_cluster_version - #WARNING: Avoid using this option (cluster_endpoint_public_access = true) in preprod or prod accounts. This feature is designed for sandbox accounts, simplifying cluster deployment and testing. - cluster_endpoint_public_access = true - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false - - manage_aws_auth_configmap = true - aws_auth_roles = [ - { - # Required for EMR on EKS virtual cluster - rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/AWSServiceRoleForAmazonEMRContainers" - username = "emr-containers" - }, - ] - - fargate_profiles = { - emr_wildcard = { - selectors = [ - { namespace = "emr-*" } - ] - } - kube_system = { - name = "kube-system" - selectors = [ - { namespace = "kube-system" } - ] - } - } - - tags = local.tags -} diff --git a/analytics/terraform/emr-eks-fargate/outputs.tf b/analytics/terraform/emr-eks-fargate/outputs.tf deleted file mode 100644 index 1a3c5b544..000000000 --- a/analytics/terraform/emr-eks-fargate/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" -} - -output "emr_on_eks" { - description = "EMR on EKS" - value = module.emr_containers -} diff --git a/analytics/terraform/emr-eks-fargate/variables.tf b/analytics/terraform/emr-eks-fargate/variables.tf deleted file mode 100644 index f3177ba5c..000000000 --- a/analytics/terraform/emr-eks-fargate/variables.tf +++ /dev/null @@ -1,42 +0,0 @@ -variable "name" { - description = "Name of the VPC and EKS Cluster" - default = "emr-eks-fargate" - type = string -} - -variable "region" { - description = "Region" - default = "us-west-2" - type = string -} - -variable "eks_cluster_version" { - description = "EKS Cluster version" - type = string - default = "1.27" -} - -variable "tags" { - description = "Default tags" - type = map(string) - default = {} -} - -variable "vpc_cidr" { - description = "VPC CIDR" - type = string - default = "10.1.0.0/16" -} - -# Only two Subnets for with low IP range for internet access -variable "public_subnets" { - description = "Public Subnets CIDRs. 62 IPs per Subnet" - type = list(string) - default = ["10.1.255.128/26", "10.1.255.192/26"] -} - -variable "private_subnets" { - description = "Private Subnets CIDRs. 32766 Subnet1 and 16382 Subnet2 IPs per Subnet" - type = list(string) - default = ["10.1.0.0/17", "10.1.128.0/18"] -} diff --git a/analytics/terraform/emr-eks-fargate/versions.tf b/analytics/terraform/emr-eks-fargate/versions.tf deleted file mode 100644 index bbb7aa51d..000000000 --- a/analytics/terraform/emr-eks-fargate/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.47" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4" - } - } - - # ## Used for end-to-end testing on project; update to suit your needs - # backend "s3" { - # bucket = "doeks-github-actions-e2e-test-state" - # region = "us-west-2" - # key = "e2e/emr-eks-fargate/terraform.tfstate" - # } -} diff --git a/analytics/terraform/emr-eks-fargate/vpc.tf b/analytics/terraform/emr-eks-fargate/vpc.tf deleted file mode 100644 index bc7e385aa..000000000 --- a/analytics/terraform/emr-eks-fargate/vpc.tf +++ /dev/null @@ -1,38 +0,0 @@ -#--------------------------------------------------------------- -# Supporting Network Resources -#--------------------------------------------------------------- -# WARNING: This VPC module includes the creation of an Internet Gateway and NAT Gateway, which simplifies cluster deployment and testing, primarily intended for sandbox accounts. -# IMPORTANT: For preprod and prod use cases, it is crucial to consult with your security team and AWS architects to design a private infrastructure solution that aligns with your security requirements - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = local.name - cidr = local.vpc_cidr - azs = local.azs - - public_subnets = var.public_subnets - private_subnets = var.private_subnets - - enable_nat_gateway = true - single_nat_gateway = true - - # Manage so we can name - manage_default_network_acl = true - default_network_acl_tags = { Name = "${local.name}-default" } - manage_default_route_table = true - default_route_table_tags = { Name = "${local.name}-default" } - manage_default_security_group = true - default_security_group_tags = { Name = "${local.name}-default" } - - public_subnet_tags = { - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} diff --git a/distributed-databases/cassandra/README.md b/distributed-databases/cassandra/README.md deleted file mode 100755 index 455284ca7..000000000 --- a/distributed-databases/cassandra/README.md +++ /dev/null @@ -1 +0,0 @@ -# Cassandra on EKS (Coming Soon) diff --git a/distributed-databases/cockroachdb/README.md b/distributed-databases/cockroachdb/README.md deleted file mode 100755 index 9d3ba7eb6..000000000 --- a/distributed-databases/cockroachdb/README.md +++ /dev/null @@ -1 +0,0 @@ -# CockroachDB on EKS (Coming Soon) diff --git a/distributed-databases/mongodb/README.md b/distributed-databases/mongodb/README.md deleted file mode 100755 index 316c97b3c..000000000 --- a/distributed-databases/mongodb/README.md +++ /dev/null @@ -1 +0,0 @@ -# MongoDB on EKS (Coming Soon) diff --git a/website/docs/blueprints/ai-ml/jark.md b/website/docs/blueprints/ai-ml/jark.md new file mode 100644 index 000000000..0396514da --- /dev/null +++ b/website/docs/blueprints/ai-ml/jark.md @@ -0,0 +1,147 @@ +--- +sidebar_position: 2 +sidebar_label: JARK on EKS +--- +import CollapsibleContent from '../../../src/components/CollapsibleContent'; + +# JARK on EKS + +:::caution +This blueprint should be considered as experimental and should only be used for proof of concept. +::: + +:::info +As part of our ongoing efforts to make this blueprint more enterprise-ready, we are actively working on adding several key functionalities. This includes cost management with Kubecost, advanced observability with OTEL, Amazon Managed Prometheus, and Grafana, as well as improved security and data governance using tools such as OPA/Gatekeeper and IRSA. If you have specific requirements or suggestions for this blueprint, please feel free to open an issue on our GitHub repository. +::: + +## Introduction + +[Ray](https://www.ray.io/) is an open-source framework for building scalable and distributed applications. It is designed to make it easy to write parallel and distributed Python applications by providing a simple and intuitive API for distributed computing. It has a growing community of users and contributors, and is actively maintained and developed by the Ray team at Anyscale, Inc. + +To deploy Ray in production across multiple machines users must first deploy [**Ray Cluster**](https://docs.ray.io/en/latest/cluster/getting-started.html). A Ray Cluster consists of head nodes and worker nodes which can be autoscaled using the built-in **Ray Autoscaler**. + +![RayCluster](img/ray-cluster.svg) + +*Source: https://docs.ray.io/en/latest/cluster/key-concepts.html* + +## Ray on Kubernetes + +Deploying Ray Cluster on Kubernetes including on Amazon EKS is supported via the [**KubeRay Operator**](https://ray-project.github.io/kuberay/). The operator provides a Kubernetes-native way to manage Ray clusters. The installation of KubeRay Operator involves deploying the operator and the CRDs for `RayCluster`, `RayJob` and `RayService` as documented [here](https://ray-project.github.io/kuberay/deploy/helm/). + +Deploying Ray on Kubernetes can provide several benefits: + +1. Scalability: Kubernetes allows you to scale your Ray cluster up or down based on your workload requirements, making it easy to manage large-scale distributed applications. + +1. Fault tolerance: Kubernetes provides built-in mechanisms for handling node failures and ensuring high availability of your Ray cluster. + +1. Resource allocation: With Kubernetes, you can easily allocate and manage resources for your Ray workloads, ensuring that they have access to the necessary resources for optimal performance. + +1. Portability: By deploying Ray on Kubernetes, you can run your workloads across multiple clouds and on-premises data centers, making it easy to move your applications as needed. + +1. Monitoring: Kubernetes provides rich monitoring capabilities, including metrics and logging, making it easy to troubleshoot issues and optimize performance. + +Overall, deploying Ray on Kubernetes can simplify the deployment and management of distributed applications, making it a popular choice for many organizations that need to run large-scale machine learning workloads. + +Before moving forward with the deployment please make sure you have read the pertinent sections of the official [documentation](https://docs.ray.io/en/latest/cluster/kubernetes/index.html). + +![RayonK8s](img/ray_on_kubernetes.webp) + +*Source: https://docs.ray.io/en/latest/cluster/kubernetes/index.html* + +## Deploying the Example + +In this [example](https://github.com/awslabs/data-on-eks/tree/main/ai-ml/ray/terraform), you will provision Ray Cluster on Amazon EKS using the KubeRay Operator. The example also demonstrates the use of Karpenter of autoscaling of worker nodes for job specific Ray Clusters. + + +![RayOnEKS](img/ray-on-eks.png) + +Pre-requisites}> + +Ensure that you have installed the following tools on your machine. + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) +4. [python3](https://www.python.org/) +6. [ray](https://docs.ray.io/en/master/ray-overview/installation.html#from-wheels) + + + +Deploy the EKS Cluster with KubeRay Operator}> + +#### Clone the repository + +```bash +git clone https://github.com/awslabs/data-on-eks.git +``` + +Navigate into one of the example directories and run `install.sh` script + +**Important Note:** Ensure that you update the region in the `variables.tf` file before deploying the blueprint. +Additionally, confirm that your local region setting matches the specified region to prevent any discrepancies. +For example, set your `export AWS_DEFAULT_REGION=""` to the desired region: + +```bash +cd data-on-eks/ai-ml/jark-stack/ && chmod +x install.sh +./install.sh +``` + + + +Verify Deployment}> + +Update local kubeconfig so we can access kubernetes cluster + +```bash +aws eks update-kubeconfig --name jark-stack #or whatever you used for EKS cluster name +``` + +First, lets verify that we have worker nodes running in the cluster. + +```bash +kubectl get nodes +``` + +```bash +NAME STATUS ROLES AGE VERSION +ip-10-1-26-241.ec2.internal Ready 10h v1.24.9-eks-49d8fe8 +ip-10-1-4-21.ec2.internal Ready 10h v1.24.9-eks-49d8fe8 +ip-10-1-40-196.ec2.internal Ready 10h v1.24.9-eks-49d8fe8 +``` + + +Next, lets verify all the pods are running. + +```bash +kubectl get pods -n kuberay-operator +``` + +```bash +NAME READY STATUS RESTARTS AGE +kuberay-operator-7b5c85998-vfsjr 1/1 Running 1 (1h37m ago) 1h +``` + + + +:::info + +Please check out deploying the LLM Inference or Training using the JARK stack under the [Gen AI](https://awslabs.github.io/data-on-eks/docs/gen-ai) page. + +::: + + + +Clean Up}> + +:::caution +To avoid unwanted charges to your AWS account, delete all the AWS resources created during this deployment. +::: + +This script will cleanup the environment using `-target` option to ensure all the resources are deleted in correct order. + +```bash +cd ../../../ai-ml/jark-stack/ +./cleanup.sh +``` + + diff --git a/website/docs/blueprints/ai-ml/ray.md b/website/docs/blueprints/ai-ml/ray.md deleted file mode 100644 index 878ca304a..000000000 --- a/website/docs/blueprints/ai-ml/ray.md +++ /dev/null @@ -1,317 +0,0 @@ ---- -sidebar_position: 2 -sidebar_label: Ray on EKS ---- -import CollapsibleContent from '../../../src/components/CollapsibleContent'; - -# Ray on EKS - -:::caution -This blueprint should be considered as experimental and should only be used for proof of concept. -::: - -:::info -As part of our ongoing efforts to make this blueprint more enterprise-ready, we are actively working on adding several key functionalities. This includes cost management with Kubecost, advanced observability with OTEL, Amazon Managed Prometheus, and Grafana, as well as improved security and data governance using tools such as OPA/Gatekeeper and IRSA. If you have specific requirements or suggestions for this blueprint, please feel free to open an issue on our GitHub repository. -::: - -## Introduction - -[Ray](https://www.ray.io/) is an open-source framework for building scalable and distributed applications. It is designed to make it easy to write parallel and distributed Python applications by providing a simple and intuitive API for distributed computing. It has a growing community of users and contributors, and is actively maintained and developed by the Ray team at Anyscale, Inc. - -To deploy Ray in production across multiple machines users must first deploy [**Ray Cluster**](https://docs.ray.io/en/latest/cluster/getting-started.html). A Ray Cluster consists of head nodes and worker nodes which can be autoscaled using the built-in **Ray Autoscaler**. - -![RayCluster](img/ray-cluster.svg) - -*Source: https://docs.ray.io/en/latest/cluster/key-concepts.html* - -## Ray on Kubernetes - -Deploying Ray Cluster on Kubernetes including on Amazon EKS is supported via the [**KubeRay Operator**](https://ray-project.github.io/kuberay/). The operator provides a Kubernetes-native way to manage Ray clusters. The installation of KubeRay Operator involves deploying the operator and the CRDs for `RayCluster`, `RayJob` and `RayService` as documented [here](https://ray-project.github.io/kuberay/deploy/helm/). - -Deploying Ray on Kubernetes can provide several benefits: - -1. Scalability: Kubernetes allows you to scale your Ray cluster up or down based on your workload requirements, making it easy to manage large-scale distributed applications. - -1. Fault tolerance: Kubernetes provides built-in mechanisms for handling node failures and ensuring high availability of your Ray cluster. - -1. Resource allocation: With Kubernetes, you can easily allocate and manage resources for your Ray workloads, ensuring that they have access to the necessary resources for optimal performance. - -1. Portability: By deploying Ray on Kubernetes, you can run your workloads across multiple clouds and on-premises data centers, making it easy to move your applications as needed. - -1. Monitoring: Kubernetes provides rich monitoring capabilities, including metrics and logging, making it easy to troubleshoot issues and optimize performance. - -Overall, deploying Ray on Kubernetes can simplify the deployment and management of distributed applications, making it a popular choice for many organizations that need to run large-scale machine learning workloads. - -Before moving forward with the deployment please make sure you have read the pertinent sections of the official [documentation](https://docs.ray.io/en/latest/cluster/kubernetes/index.html). - -![RayonK8s](img/ray_on_kubernetes.webp) - -*Source: https://docs.ray.io/en/latest/cluster/kubernetes/index.html* - -## Deploying the Example - -In this [example](https://github.com/awslabs/data-on-eks/tree/main/ai-ml/ray/terraform), you will provision Ray Cluster on Amazon EKS using the KubeRay Operator. The example also demonstrates the use of Karpenter of autoscaling of worker nodes for job specific Ray Clusters. - - -![RayOnEKS](img/ray-on-eks.png) - -Pre-requisites}> - -Ensure that you have installed the following tools on your machine. - -1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) -3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) -4. [python3](https://www.python.org/) -6. [ray](https://docs.ray.io/en/master/ray-overview/installation.html#from-wheels) - - - -Deploy the EKS Cluster with KubeRay Operator}> - -#### Clone the repository - -```bash -git clone https://github.com/awslabs/data-on-eks.git -``` - -#### Initialize Terraform - -Navigate into the example directory - -```bash -cd data-on-eks/ai-ml/ray/terraform -``` - -#### Run the install script - - -Use the provided helper script `install.sh` to run the terraform init and apply commands. By default the script deploys EKS cluster to `us-west-2` region. Update `variables.tf` to change the region. This is also the time to update any other input variables or make any other changes to the terraform template. - - -```bash -./install .sh -``` - - - -Verify Deployment}> - -Update local kubeconfig so we can access kubernetes cluster - -```bash -aws eks update-kubeconfig --name ray-cluster #or whatever you used for EKS cluster name -``` - -First, lets verify that we have worker nodes running in the cluster. - -```bash -kubectl get nodes -``` -:::info -```bash -NAME STATUS ROLES AGE VERSION -ip-10-1-26-241.ec2.internal Ready 10h v1.24.9-eks-49d8fe8 -ip-10-1-4-21.ec2.internal Ready 10h v1.24.9-eks-49d8fe8 -ip-10-1-40-196.ec2.internal Ready 10h v1.24.9-eks-49d8fe8 -``` -::: - -Next, lets verify all the pods are running. - -```bash -kubectl get pods -n kuberay-operator -``` -:::info -```bash -NAME READY STATUS RESTARTS AGE -kuberay-operator-7b5c85998-vfsjr 1/1 Running 1 (1h37m ago) 1h -``` -::: - - -At this point we are ready to deploy Ray Clusters. - - -Deploy Ray Clusters and Workloads}> - -For convenience, we have packaged the helm chart deployent of Ray Cluster as a repeatable terraform [module](https://github.com/awslabs/data-on-eks/tree/main/ai-ml/ray/terraform/modules/ray-cluster/). This allows us to codify organizational best practices and requirements for deploying Ray Clusters for multiple Data Science teams. The module also creates configuration needed for karpenter to be able to provision EC2 instances for Ray applications as and when they are needed for the duration of the job. This model can be replicated via GitOps tooling such as ArgoCD or Flux but is done here via terraform for demonstration purpose. - -##### XGBoost - -First, we will deploy a Ray Cluster for our [XGBoost benchmark](https://docs.ray.io/en/latest/cluster/kubernetes/examples/ml-example.html#kuberay-ml-example) sample job. - -Go to the xgboost directory followed by terraform init, and plan. - -```bash -cd examples/xgboost -terraform init -terraform plan -``` - -If the changes look good, lets apply them. - -```bash -terraform apply -auto-approve -``` - -As the RayCluster pod goes into the pending state, Karpenter will provision an EC2 instance based on the `Provisioner` and `AWSNodeTemplate` configuration we have provided. We can check that a new node has been created. - -```bash -kubectl get nodes -``` - -:::info -```bash -NAME STATUS ROLES AGE VERSION -# New node appears -ip-10-1-13-204.ec2.internal Ready 2m22s v1.24.9-eks-49d8fe8 -ip-10-1-26-241.ec2.internal Ready 12h v1.24.9-eks-49d8fe8 -ip-10-1-4-21.ec2.internal Ready 12h v1.24.9-eks-49d8fe8 -ip-10-1-40-196.ec2.internal Ready 12h v1.24.9-eks-49d8fe8 -``` -::: - -Wait until the RayCluster head node pods are provisioned. - -```bash -kubectl get pods -n xgboost -``` -:::info -``` -NAME READY STATUS RESTARTS AGE -xgboost-kuberay-head-585d6 2/2 Running 0 5m42s -``` -::: - -Now we are ready to run our sample training benchmark using for XGBoost. First, open another terminal and forward the Ray server to our localhost. - -```sh -kubectl port-forward service/xgboost-kuberay-head-svc -n xgboost 8265:8265 -``` -:::info -```bash -Forwarding from 127.0.0.1:8265 -> 8265 -Forwarding from [::1]:8265 -> 8265 -``` -::: - -Submit the ray job for XGBoost benchmark. - -```bash -python job/xgboost_submit.py -``` - -You can open http://localhost:8265 in your browser to monitor job progress. If there are any failures during execution those can be viewed in the logs under the Jobs section. - -![RayDashboard](img/ray-dashboard.png) - -As the job progresses, you will notice new Ray autoscaler will provision additional ray worker pods based on the autoscaling configuration defined in the RayCluster configuration. Those worker pods will initially remain in pending state. That will trigger karpenter to spin up new EC2 instances so the pending pods can be scheduled. After worker pods go to running state, the job will progress to completion. - -```bash -kubectl get nodes -``` -:::info -```bash -NAME STATUS ROLES AGE VERSION -ip-10-1-1-241.ec2.internal Unknown 1s -ip-10-1-10-211.ec2.internal Unknown 1s -ip-10-1-13-204.ec2.internal Ready 24m v1.24.9-eks-49d8fe8 -ip-10-1-26-241.ec2.internal Ready 12h v1.24.9-eks-49d8fe8 -ip-10-1-3-64.ec2.internal Unknown 7s -ip-10-1-4-21.ec2.internal Ready 12h v1.24.9-eks-49d8fe8 -ip-10-1-40-196.ec2.internal Ready 12h v1.24.9-eks-49d8fe8 -ip-10-1-7-167.ec2.internal Unknown 1s -ip-10-1-9-112.ec2.internal Unknown 1s -ip-10-1-9-172.ec2.internal Unknown 1s -``` -::: - -Optionally, you can also use [eks-node-viewer](https://github.com/awslabs/eks-node-viewer) for visualizing dynamic node usage within the cluster. - -![EksNodeViewer](img/eks-node-viewer.png) - -Once the benchmark is complete, the job log will display the results. You might see different results based on your configurations. - -:::info -```bash -Results: {'training_time': 1338.488839321999, 'prediction_time': 403.36653568099973} -``` -::: -##### PyTorch - -We can simultaneously deploy the PyTorch benchmark as well. We deploy a separate Ray Cluster with its own configuration for Karpenter workers. Different jobs can have different requirements for Ray Cluster such as a different version of Ray libraries or EC2 instance configuration such as making use of Spot market or GPU instances. We take advantage of node taints and tolerations in Ray Cluster pod specs to match the Ray Cluster configuration to Karpenter configuration thus taking advantage of the flexibility that Karpenter provides. - -Go to the PyTorch directory and run the terraform init and plan as before. - -```bash -cd ../pytorch -terraform init -terraform plan -``` - -Apply the changes. - - -```bash -terraform apply -auto-approve -``` - -Wait for the pytorch Ray Cluster head node pods to be ready. - -```bash -kubectl get pods -n pytorch -w -``` - -:::info -```bash -NAME READY STATUS RESTARTS AGE -pytorch-kuberay-head-9tx56 0/2 Pending 0 43s -``` -::: - -Once running, we can forward the port for server, taking care that we forward it to another local port as 8265 may be occupied by the xgboost connection. - -```bash -kubectl port-forward service/pytorch-kuberay-head-svc -n pytorch 8266:8265 -``` - -We can then submit the job for PyTorch benchmark workload. - -```bash -python job/pytorch_submit.py -``` - -You can open http://localhost:8266 to monitor the progress of the pytorch benchmark. - - -Teardown}> - -:::caution -To avoid unwanted charges to your AWS account, delete all the AWS resources created during this deployment. -::: - -Destroy the Ray Clusters for pytorch followed by xgboost. - -From the pytorch directory. - -```bash -cd ../pytorch -terraform destroy -auto-approve -``` - -From the xgboost directory. - -```bash -cd ../xgboost -terraform destroy -auto-approve -``` - -Use the provided helper script `cleanup.sh` to tear down EKS cluster and other AWS resources. - -```bash -cd ../../ -./cleanup.sh -``` - - diff --git a/website/docs/blueprints/ai-ml/trainium.md b/website/docs/blueprints/ai-ml/trainium.md index e10314877..033907e04 100644 --- a/website/docs/blueprints/ai-ml/trainium.md +++ b/website/docs/blueprints/ai-ml/trainium.md @@ -1,10 +1,10 @@ --- -sidebar_position: 5 +sidebar_position: 2 sidebar_label: Trainium on EKS --- import CollapsibleContent from '../../../src/components/CollapsibleContent'; -# AWS Trainium on EKS +# AWS Trainium and Inferentia on EKS [AWS Trainium](https://aws.amazon.com/machine-learning/trainium/) is an advanced ML accelerator that transforms high-performance deep learning(DL) training. `Trn1` instances, powered by AWS Trainium chips, are purpose-built for high-performance DL training of **100B+ parameter** models. Meticulously designed for exceptional performance, Trn1 instances cater specifically to training popular Natual Language Processing(NLP) models on AWS, offering up to **50% cost savings ** compared to GPU-based EC2 instances. This cost efficiency makes them an attractive option for data scientists and ML practitioners seeking optimized training costs without compromising performance. At the core of Trn1 instance's capabilities lies the [AWS Neuron SDK](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/), a software development kit seamlessly integrated with leading ML frameworks and libraries, such as [PyTorch](https://pytorch.org/), [TensorFlow](https://tensorflow.org/), [Megatron-LM](https://huggingface.co/docs/accelerate/usage_guides/megatron_lm), and [Hugging Face](https://huggingface.co/). The Neuron SDK empowers developers to train NLP, computer vision, and recommender models on Trainium with ease, requiring only a few lines of code changes. diff --git a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-ack.md b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-ack.md deleted file mode 100644 index dc2b61025..000000000 --- a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-ack.md +++ /dev/null @@ -1,153 +0,0 @@ ---- -sidebar_position: 4 -sidebar_label: EMR on EKS with ACK Controller ---- -import CollapsibleContent from '../../../src/components/CollapsibleContent'; - -# ACK Controller for EMR on EKS - -## Introduction -In this post, we will learn to build EMR on EKS Spark workloads by using [AWS Controllers for Kubernetes (ACK)](https://aws-controllers-k8s.github.io/community/docs/tutorials/emr-on-eks-example/). -We will also build a end-to-end observability for Spark workloads by leveraging Amazon Managed Service for Prometheus to collect and store the metrics generated by Spark Applications and then use Amazon Managed Grafana to build dashboards for monitoring use cases. - -Deploying the Solution}> - -In this [example](https://github.com/awslabs/data-on-eks/tree/main/analytics/terraform/emr-eks-ack), you will provision the following resources required to run Spark Jobs using EMR on EKS, as well as monitor spark job metrics using **Amazon Managed Prometheus** and **Amazon Managed Grafana**. - -- Creates EKS Cluster Control plane with public endpoint (for demo purpose only) -- Two managed node groups - - Core Node group with 3 AZs for running system critical pods. e.g., Cluster Autoscaler, CoreDNS, Observability, Logging etc. - - Spark Node group with single AZ for running Spark jobs -- Enable EMR on EKS and creates two Data teams (`emr-data-team-a`, `emr-data-team-b`) - - Creates new namespace for each team - - Creates Kubernetes role and role binding(`emr-containers` user) for the above namespace - - New IAM role for the team execution role - - Update AWS_AUTH config map with emr-containers user and AWSServiceRoleForAmazonEMRContainers role - - Create a trust relationship between the job execution role and the identity of the EMR managed service account -- EMR Virtual Cluster for `emr-data-team-a` -- IAM policy for `emr-data-team-a` -- Amazon Managed Prometheus workspace to remote write metrics from Prometheus server -- Deploys the following Kubernetes Add-ons - - Managed Add-ons - - VPC CNI, CoreDNS, KubeProxy, AWS EBS CSi Driver - - Self Managed Add-ons - - Metrics server with HA, CoreDNS Cluster proportional Autoscaler, Cluster Autoscaler, Prometheus Server and Node Exporter, VPA for Prometheus, AWS for FluentBit, CloudWatchMetrics for EKS - - ACK EMR containers controller which allows you deploy sparks jobs to EMR on EKS by using yaml files. The controller is installed by using [AWS EKS ACK Addons Terraform module](https://github.com/aws-ia/terraform-aws-eks-ack-addons) - -### Prerequisites: - -Ensure that you have installed the following tools on your machine. - -1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) -3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) - -_Note: Currently Amazon Managed Prometheus supported only in selected regions. Please see this [userguide](https://docs.aws.amazon.com/prometheus/latest/userguide/what-is-Amazon-Managed-Service-Prometheus.html) for supported regions._ - -### Deploy - -Clone the repository - -```bash -git clone https://github.com/awslabs/data-on-eks.git -``` - -Navigate into one of the example directories and run `terraform init` - -```bash -cd data-on-eks/analytics/terraform/emr-eks-ack -terraform init -``` - -Set `AWS_REGION` and Run Terraform plan to verify the resources created by this execution. - -```bash -export AWS_REGION="us-west-2" # Change region according to your needs -terraform plan -``` - -Deploy the pattern - -```bash -terraform apply -``` - -Enter `yes` to apply. - -## Verify the resources - -Let’s verify the resources created by `terraform apply`. - -Verify the Amazon EKS Cluster and Amazon Managed service for Prometheus. - -```bash -aws eks describe-cluster --name emr-eks-ack - -aws amp list-workspaces --alias amp-ws-emr-eks-ack -``` - -Verify EMR on EKS Namespaces `emr-data-team-a` and `emr-data-team-b` and Pod status for `Prometheus`, `Vertical Pod Autoscaler`, `Metrics Server` and `Cluster Autoscaler`. - -```bash -aws eks --region us-west-2 update-kubeconfig --name emr-eks-ack # Creates k8s config file to authenticate with EKS Cluster - -kubectl get nodes # Output shows the EKS Managed Node group nodes - -kubectl get ns | grep emr-data-team # Output shows emr-data-team-a and emr-data-team-b namespaces for data teams - -kubectl get pods --namespace=prometheus # Output shows Prometheus server and Node exporter pods - -kubectl get pods --namespace=vpa # Output shows Vertical Pod Autoscaler pods - -kubectl get pods --namespace=kube-system | grep metrics-server # Output shows Metric Server pod - -kubectl get pods --namespace=kube-system | grep cluster-autoscaler # Output shows Cluster Autoscaler pod -``` - - - -### Setup Amazon Managed Grafana with SSO -Currently, this step is manual. Please follow the steps in this [blog](https://aws.amazon.com/blogs/mt/monitoring-amazon-emr-on-eks-with-amazon-managed-prometheus-and-amazon-managed-grafana/) to create Amazon Managed Grafana with SSO enabled in your account. -You can visualize the Spark jobs runs and metrics using Amazon Managed Prometheus and Amazon Managed Grafana. - -Execute Sample Spark job - EMR Virtual Cluster}> - -We can now create EMR Virtual Cluster. An EMR Virtual Cluster is mapped to a Kubernetes namespace. EMR uses virtual clusters to run jobs and host endpoints. -Create a Virtual cluster my-ack-vc for emr-data-team-a - -```bash -kubectl apply -f analytics/terraform/emr-eks-ack/examples/emr-virtualcluster.yaml - -kubectl describe virtualclusters -``` -You will get the output like below
- ![](img/ack-virtualcluster.png) - - -Execute the following shell script to run the Spark Job. This will ask for two inputs which can be extrcated from terraform outputs. - -```bash -./analytics/terraform/emr-eks-ack/examples/sample-pyspark-job.sh - -kubectl describe jobruns -``` -You will get the output like below
- ![](img/ack-sparkjob.png) - - - -Cleanup}> - -This script will cleanup the environment using `-target` option to ensure all the resources are deleted in correct order. - -```bash -cd analytics/terraform/emr-eks-ack && chmod +x cleanup.sh -./cleanup.sh -``` - - -:::caution - -To avoid unwanted charges to your AWS account, delete all the AWS resources created during this deployment - -::: diff --git a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-cdk.md b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-cdk.md deleted file mode 100644 index a806c7868..000000000 --- a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-cdk.md +++ /dev/null @@ -1,158 +0,0 @@ ---- -sidebar_position: 5 -sidebar_label: EMR on EKS with CDK ---- - -# EMR on EKS with CDK blueprint - -## Introduction -In this post, we will learn how to use EMR on EKS AddOn and Teams in the `cdk-eks-blueprints` to deploy a an infrasturcture on EKS to submit Spark Job. The `cdk-eks-blueprints` allows you deploy an EKS cluster and enable it to be used by EMR on EKS service with minimal setup. The architecture below shows a conceptual view of the infrastructure you will deploy through this blueprint. - -![EMR on EKS CDK](img/emr-eks-cdk.png) - -## Deploying the Solution - -In this [example](https://github.com/awslabs/data-on-eks/tree/main/analytics/cdk/emr-eks), you will provision the following: - -- Creates EKS Cluster Control plane with public endpoint (for demo purpose only) -- Two managed node groups - - Core Node group with 3 AZs for running system critical pods. e.g., Cluster Autoscaler, CoreDNS, Logging etc. - - Spark Node group with single AZ for running Spark jobs -- Enable EMR on EKS and create one Data teams (`emr-data-team-a`) - - Creates new namespace for each team - - Creates Kubernetes role and role binding(`emr-containers` user) for the above namespace - - New IAM role for the team execution role - - Update AWS_AUTH config map with emr-containers user and AWSServiceRoleForAmazonEMRContainers role - - Create a trust relationship between the job execution role and the identity of the EMR managed service account -- EMR Virtual Cluster for `emr-data-team-a` -- IAM policy for `emr-data-team-a` -- Deploys the following Kubernetes Add-ons - - Managed Add-ons - - VPC CNI, CoreDNS, KubeProxy, AWS EBS CSi Driver - - Self Managed Add-ons - - Metrics server with HA, Cluster Autoscaler, CertManager and AwsLoadBalancerController - -This blueprint can also take an EKS cluster that you defined using the `cdk-blueprints-library`. - -### Prerequisites - -Ensure that you have installed the following tools on your machine. - -1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) -3. [CDK](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install) - -**NOTE:** You need to have an AWS account and region that are [bootstrapped](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_bootstrap) by AWS CDK. - -### Customize - -The the entry point for this cdk blueprint is `/bin/emr-eks.ts` which instantiate a stack defined in `lib/emr-eks-blueprint-stack.ts`. This stack must be provided with a VPC and a list of EMR on EKS team definition and the role that will be admin of the EKS cluster. It can also take as options an EKS cluster defined through `cdk-blueprints-library` and the EKS cluster name. - -The properties that are passed to the emr on eks blueprint stack are defined as such: - -```typescript -export interface EmrEksBlueprintProps extends StackProps { - clusterVpc: IVpc, - clusterAdminRoleArn: ArnPrincipal - dataTeams: EmrEksTeamProps[], - eksClusterName?: string, //Default eksBlueprintCluster - eksCluster?: GenericClusterProvider, - -} -``` - -In this example we define a VPC in `lib/vpc.ts` and is instantiated in `bin/emr-eks.ts`. We also define a team called `emr-data-team-a` and which has an execution role called `myBlueprintExecRole`. -The blueprint will deploy by default an EKS cluster with the managed nodegroups defined in the section [Deploying the Solution](#deploying-the-solution). - -### Deploy - -Before you run the solution, you **MUST** change the `clusterAdminRoleArn` of the `props` object in `lib/emr-eks.ts`. This role allows you to interact manage EKS cluster and should have be allowed at least the IAM action `eks:AccessKubernetesApi`. - -Clone the repository - -```bash -git clone https://github.com/awslabs/data-on-eks.git -``` - -Navigate into one of the example directories and run `cdk synth` - -```bash -cd analytics/cdk/emr-eks -npm install -cdk synth --profile YOUR-AWS-PROFILE -``` - -Deploy the pattern - -```bash -cdk deploy --all -``` - -Enter `yes` to deploy. - -## Verify the resources - -Let’s verify the resources created by `cdk deploy`. - -Verify the Amazon EKS Cluster - -```bash -aws eks describe-cluster --name eksBlueprintCluster # Update the name cluster name if you supplied your own - -``` - -Verify EMR on EKS Namespaces `batchjob` and Pod status for `Metrics Server` and `Cluster Autoscaler`. - -```bash -aws eks --region update-kubeconfig --name eksBlueprintCluster # Creates k8s config file to authenticate with EKS Cluster. Update the name cluster name if you supplied your own - -kubectl get nodes # Output shows the EKS Managed Node group nodes - -kubectl get ns | grep batchjob # Output shows batchjob - -kubectl get pods --namespace=kube-system | grep metrics-server # Output shows Metric Server pod - -kubectl get pods --namespace=kube-system | grep cluster-autoscaler # Output shows Cluster Autoscaler pod -``` - -## Execute Sample Spark job on EMR Virtual Cluster -Execute the Spark job using the below shell script. - -- Once you deploy the blueprint you will have as output the Virtual Cluster id. You can use the id and the execution role for which you supplied a policy to submit jobs. Below you can find an example of a job you can submit with AWS CLI. - -```bash - -export EMR_ROLE_ARN=arn:aws:iam:::role/myBlueprintExecRole - -aws emr-containers start-job-run \ - --virtual-cluster-id= \ - --name=pi-2 \ - --execution-role-arn=$EMR_ROLE_ARN \ - --release-label=emr-6.8.0-latest \ - --job-driver='{ - "sparkSubmitJobDriver": { - "entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py", - "sparkSubmitParameters": "--conf spark.executor.instances=1 --conf spark.executor.memory=2G --conf spark.executor.cores=1 --conf spark.driver.cores=1 --conf spark.kubernetes.node.selector.app=spark" - } - }' - -``` - -Verify the job execution - -```bash -kubectl get pods --namespace=batchjob -w -``` - -## Cleanup - -To clean up your environment, you call the command below. This will destroy the Kubernetes Add-ons, EKS cluster with Node groups and VPC - -```bash -cdk destroy --all -``` - -:::caution - -To avoid unwanted charges to your AWS account, delete all the AWS resources created during this deployment -::: diff --git a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-fargate.md b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-fargate.md deleted file mode 100644 index 5e90f6d8b..000000000 --- a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-fargate.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -sidebar_position: 6 -sidebar_label: EMR on EKS with Fargate ---- - -# EMR Virtual Cluster on EKS Fargate - -This example shows how to provision a serverless cluster (serverless data plane) using Fargate Profiles to support EMR on EKS virtual clusters. - -There are two Fargate profiles created: -1. `kube-system` to support core Kubernetes components such as CoreDNS -2. `emr-wildcard` which supports any namespaces that begin with `emr-*`; this allows for creating multiple virtual clusters without having to create additional Fargate profiles for each new cluster. - -Using the `emr-on-eks` module, you can provision as many EMR virtual clusters as you would like by passing in multiple virtual cluster definitions to `emr_on_eks_config`. Each virtual cluster will get its own set of resources with permissions scoped to only that set of resources. The resources created by the `emr-on-eks` addon include: -- Kubernetes namespace, role, and role binding; existing or externally created namespace and role can be utilized as well -- IAM role for service account (IRSA) used by for job execution. Users can scope access to the appropriate S3 bucket and path via `s3_bucket_arns`, use for both accessing job data as well as writing out results. The bare minimum permissions have been provided for the job execution role; users can provide additional permissions by passing in additional policies to attach to the role via `iam_role_additional_policies` -- CloudWatch log group for task execution logs. Log streams are created by the job itself and not via Terraform -- EMR managed security group for the virtual cluster -- EMR virtual cluster scoped to the namespace created/provided - -To learn more about running completely serverless EKS clusters using Fargate, see the [`fargate-serverless`](https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/main/examples/fargate-serverless#serverless-eks-cluster-using-fargate-profiles) example. - -:::info - -Please be informed that the method of creating EMR on EKS clusters has changed and is now done as a Kubernetes add-on. -This differs from previous blueprints which deployed EMR on EKS as part of the EKS Cluster module. -Our team is working towards simplifying both deployment approaches and will soon create a standalone Terraform module for this purpose. -Additionally, all blueprints will be updated with this new dedicated EMR on EKS Terraform module. - -::: - -## Prerequisites: - -Ensure that you have the following tools installed locally: - -1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) -3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) - -### Deploy - -Clone the repository - -```bash -git clone https://github.com/awslabs/data-on-eks.git -``` - -Navigate into one of the example directories and run `terraform init` - -```bash -cd data-on-eks/analytics/emr-eks-fargate -terraform init -``` - -Set `AWS_REGION` and Run`terraform plan` to verify the resources created by this execution. - -```bash -export AWS_REGION="us-west-2" # Change according to your need -terraform plan -``` - -Deploy the pattern - -```bash -terraform apply -``` - -Enter `yes` at command prompt to apply - -## Validate - -The following command will update the `kubeconfig` on your local machine and allow you to interact with your EKS Cluster using `kubectl`. - -1. Run `update-kubeconfig` command: - -```sh -aws eks --region update-kubeconfig --name -``` - -2. Test by listing all the pods running currently. Note: the EMR on EKS virtual cluster(s) will create pods as needed to execute jobs and the pods shown will vary depending on how long after deploying the example you run the `kubectl get pods -A` command: - -```sh -kubectl get pods -A - -# Output should look like below -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system cluster-proportional-autoscaler-coredns-6ccfb4d9b5-sjb8m 1/1 Running 0 8m27s -kube-system coredns-7c8d74d658-9cmn2 1/1 Running 0 8m27s -kube-system coredns-7c8d74d658-pmf5l 1/1 Running 0 7m38s -``` - -3. Execute the sample EMR on EKS job. This will calculate the value of Pi using sample PySpark job. -```sh -cd analytics/terraform/emr-eks-fargate/examples -./basic-pyspark-job '' '' -``` - -4. Once the job is complete, navigate to the CloudWatch log console and find the log group created by this example `/emr-on-eks-logs/emr-workload/emr-workload`. Click `Search Log Group` and enter `roughly` into the search field. You should see a log entry that has the returned results from the job. - -```json -{ - "message": "Pi is roughly 3.146360", - "time": "2022-11-20T16:46:59+00:00" -} -``` - -## Destroy - -To teardown and remove the resources created in this example: - -```sh -kubectl delete all --all -n emr-workload -n emr-custom # ensure all jobs resources are cleaned up first -terraform destroy -target="module.eks_blueprints_kubernetes_addons" -auto-approve -terraform destroy -target="module.eks" -auto-approve -terraform destroy -auto-approve -``` - -If the EMR virtual cluster fails to delete and the following error is shown: -``` -Error: waiting for EMR Containers Virtual Cluster (xwbc22787q6g1wscfawttzzgb) delete: unexpected state 'ARRESTED', wanted target ''. last error: %!s() -``` - -You can clean up any of the clusters in the `ARRESTED` state with the following: - -```sh -aws emr-containers list-virtual-clusters --region us-west-2 --states ARRESTED \ ---query 'virtualClusters[0].id' --output text | xargs -I{} aws emr-containers delete-virtual-cluster \ ---region us-west-2 --id {} -``` diff --git a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-observability.md b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-observability.md index b7a51fd1d..4546f3a65 100644 --- a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-observability.md +++ b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-observability.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 3 sidebar_label: EMR on EKS Observability --- diff --git a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-spark-operator.md b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-spark-operator.md index f4022773d..6a4131b4c 100644 --- a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-spark-operator.md +++ b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-spark-operator.md @@ -1,5 +1,5 @@ --- -sidebar_position: 3 +sidebar_position: 4 sidebar_label: EMR Runtime with Spark Operator --- import CollapsibleContent from '../../../src/components/CollapsibleContent'; diff --git a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-studio.md b/website/docs/blueprints/amazon-emr-on-eks/emr-eks-studio.md deleted file mode 100644 index c16c15899..000000000 --- a/website/docs/blueprints/amazon-emr-on-eks/emr-eks-studio.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -sidebar_position: 6 -sidebar_label: EMR on EKS Data Platform with AWS CDK ---- - -# EMR on EKS Data Platform with AWS CDK - -In this document we will show you how you can use AWS CDK and the [Analytics Reference Architecture](https://aws.amazon.com/blogs/opensource/adding-cdk-constructs-to-the-aws-analytics-reference-architecture/) (ARA) library to deploy an end to end data analytics platform. This platform will allow you to run Spark interactive Session in Jupyter notebook with EMR Studio supported by EMR on EKS and run Spark jobs with EMR on EKS. The architecture below shows the infrasturcture you will deploy using the CDK and ARA library. - -![emr-eks-studio-ara-architecture](./img/emr-eks-studio-cdk-ara.png) - -## [Analytics Reference Architecture](https://aws.amazon.com/blogs/opensource/adding-cdk-constructs-to-the-aws-analytics-reference-architecture/) - -AWS Analytics Reference Architecture (ARA) exposes set of reusable core components in an AWS CDK library, currently available in Typescript and Python. This library contains AWS CDK constructs (L3) that can be used to quickly provision analytics solutions in demos, prototypes, proofs of concept, and end-to-end reference architectures. The API of ARA Library is defined [here](https://constructs.dev/packages/aws-analytics-reference-architecture/v/2.4.11?lang=typescript). - -In our case the library help you deploy an infrastructure optimised for Apache Spark running on EKS leveraging EMR on EKS. The infrastructure will out of the box provide you with pod collocation to reduce network traffic, deploy nodegroup in a single AZ to reduce cross AZ traffic during shuffle, use dedicated instances for EMR on EKS, use optimized instances for memory intensive jobs, use spot and on-demand instances for non-critical job and for critical jobs. - -## Prerequisites - -Ensure that you have installed the following tools on your machine. - -1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) -3. [CDK](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install) - -## Solution - -To deploy the data platform we will use an example in the `Analytics Reference Architecture`. The example is in the directory `examples/emr-eks-app` that you will find in the repository you will clone below. - -Clone the repository - -```bash -git clone https://github.com/aws-samples/aws-analytics-reference-architecture.git -``` - -This solution will deploy the following: - -- EKS cluster and a set of Nodegroups: - -- Managed Nodegroup called tooling for running system critical pods. e.g., Cluster Autoscaler, CoreDNS, EBS CSI Driver.. -- Three Managed Nodegroup called critical for critical jobs, each in one AZ, this nodegroup use on-demand instances -- Three Managed Nodegroup called non-critical for non-critical jobs, each in one AZ, this nodegroup use spot instances -- Three Managed Nodegroup called notebook-driver for non-critical jobs, each in one AZ, this nodegroup use on-demand instances to have a stable driver. -- Three Managed Nodegroup called notebook-executor for non-critical jobs, each in one AZ, this nodegroup use spot instances instances for executors. - -- Enable EKS Cluster to be with with EMR on EKS service -- EMR Virtual Cluster called `batchjob`, used to submitted jobs -- EMR Virtual Cluster called `emrvcplatform`, used to submitted jobs -- EMR Studio called `platform` -- A `managed endpoint`, called `platform-myendpoint` , to use with Jupyter notebooks you will create in the EMR Studio -- [Execution role](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/iam-execution-role.html) to use when submitting jobs with EMR on EKS `start-job-run` -- Execution role to use with managed endpoint. -- pod templates stored in an S3 bucket called "EKS-CLUSTER-NAME-emr-eks-assets-ACCOUNT-ID-REGION" - -### Customize - -The infrastructure described above is defined in `emr-eks-app/lib/emr-eks-app-stack.ts`. If you want to customize it you can change the values in it. For example, you can chose not to create the default nodegroup to use for `jobs`, in this case you can set the `defaultNodeGroups` parameter to `false` in the `EmrEksCluster`. You can also call the `addEmrEksNodegroup` method to define your own nodegroups with specific labels, instances or taints. The `addEmrEksNodegroup` method is defined [here](https://constructs.dev/packages/aws-analytics-reference-architecture/v/2.4.11/api/EmrEksCluster?lang=typescript#addEmrEksNodegroup). - -You can also create your own execution role through the `createExecutionRole` [method](https://constructs.dev/packages/aws-analytics-reference-architecture/v/2.4.11/api/EmrEksCluster?lang=typescript#createExecutionRole) or create a managed endpoint to attach it to an EMR Studio you deployed outside of the ARA library. - -In order to simplify this example we use IAM authentication with IAM user for `EMR Studio`. If you would like to use a user in the `AWS IAM Identity Center` you can change `studioAuthMode` in the `NotebookPlatform` construct. Below you will can see the code snippet that you need to change. - -```ts -const notebookPlatform = new ara.NotebookPlatform(this, 'platform-notebook', { -emrEks: emrEks, -eksNamespace: 'dataanalysis', -studioName: 'platform', -studioAuthMode: ara.StudioAuthMode.IAM, -}); -``` - -### Deploy - -Before you run the solution, you **MUST** change the `eksAdminRoleArn` of the `props` object of `EmrEksCluster` in `lib/emr-eks-app-stack.ts`. This role allows you to interact manage EKS cluster and should have be allowed at least the IAM action `eks:AccessKubernetesApi`. You need to also change the `identityName` in the `addUser` method of the `NotebookPlatform` construct. The identityName **MUST BE** a valid IAM username that you use. Below you will can see the code snippet that you need to change. - -```ts -notebookPlatform.addUser([{ -identityName:'', -notebookManagedEndpoints: [{ -emrOnEksVersion: 'emr-6.8.0-latest', -executionPolicy: emrEksPolicy, -managedEndpointName: 'myendpoint' -}], -}]); -``` - -Last you should also update the IAM policies passed to the `createExecutionRole`, if you want to process data that is in S3 buckets that you own. - -Navigate into one of the example directories and run `cdk synth --profile YOUR-AWS-PROFILE` - -```bash -cd examples/emr-eks-app -npm install -cdk synth --profile YOUR-AWS-PROFILE -``` - -Once the synth is completed you can deploy the infrastructrue with the following command: - -```bash -cdk deploy -``` - -At the end of the deployment you will see output like follow: - -![ara-cdk-output](./img/cdk-deploy-result.png) - -In the output you will find job sample configurations with the best practices for Spark on Kubernetes like `dynamicAllocation` and `pod collocation`. - -### Job submission - -In this example we will use the `crittical-job` job configuration to submit a job using that will compute `pi` using that is part of Spark distribution. -To submit a job we will use Below you use `start-job-run` command with AWS CLI. - -Before you run the command below, make sure to change update the following parameters with the on created by your own deployment. - - - \ – The EMR virtual cluster ID, which you get from the AWS CDK output - - \ – The name of your Spark job - - \ – The execution role you created, which you get from the AWS CDK output - - \ – The Amazon S3 URI of the driver pod template, which you get from the AWS CDK output - - \ – The Amazon S3 URI of the executor pod template, which you get from the AWS CDK output - - \ – Your CloudWatch log group name - - \ – Your CloudWatch log stream prefix - -
- AWS CLI for start-job-run command - - ```bash - aws emr-containers start-job-run \ - --virtual-cluster-id CLUSTER-ID\ - --name=SPARK-JOB-NAME\ - --execution-role-arn ROLE-ARN \ - --release-label emr-6.8.0-latest \ - --job-driver '{ - "sparkSubmitJobDriver":{ - "entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py" - } - }' \ - --configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.hadoop.hive.metastore.client.factory.class": "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory", - "spark.sql.catalogImplementation": "hive", - "spark.dynamicAllocation.enabled":"true", - "spark.dynamicAllocation.minExecutors": "8", - "spark.dynamicAllocation.maxExecutors": "40", - "spark.kubernetes.allocation.batch.size": "8", - "spark.executor.cores": "8", - "spark.kubernetes.executor.request.cores": "7", - "spark.executor.memory": "28G", - "spark.driver.cores": "2", - "spark.kubernetes.driver.request.cores": "2", - "spark.driver.memory": "6G", - "spark.dynamicAllocation.executorAllocationRatio": "1", - "spark.dynamicAllocation.shuffleTracking.enabled": "true", - "spark.dynamicAllocation.shuffleTracking.timeout": "300s", - "spark.kubernetes.driver.podTemplateFile": "s3://EKS-CLUSTER-NAME-emr-eks-assets-ACCOUNT-ID-REGION/EKS-CLUSTER-NAME/pod-template/critical-driver.yaml", - "spark.kubernetes.executor.podTemplateFile": "s3://EKS-CLUSTER-NAME-emr-eks-assets-ACCOUNT-ID-REGION/EKS-CLUSTER-NAME/pod-template/critical-executor.yaml" - } - } - ], - "monitoringConfiguration": { - "cloudWatchMonitoringConfiguration": { - "logGroupName": "Log_Group_Name", - "logStreamNamePrefix": "Log_Stream_Prefix" - } - } - }' - ``` -
- -Verify the job execution - -```bash -kubectl get pods --namespace=batchjob -w -``` - -### Interactive session - -To use an interactive session, you should log in to the EMR Studio instance with the URL provided to you at the end of `cdk deploy`. -This link will be in the form of `https://es-xxxxx/emrstudio-prod-REGION.amazonaws.com`. -Once you click on the link you will be see a log in page where you **MUST** sign-in with the username provided to the `addUser` method. When you sign in you should follow these steps. - -1. Create workspace, this will start for a Jupyter notebook -2. Connect to the Jupter notebook -3. Attach to a Virtual cluster, this would be have the following name "emrvcplatform" and chose an endpoint called "platform-myendpoint" -4. Open a notebook and select the PySpark kernel -5. You are now ready to perform analyse your data with Spark running on EMR on EKS. - -## Cleanup - -To clean up your environment, you call the command below. This will destroy the EKS cluster with Node groups and VPC - -```bash -cdk destroy -``` - -:::caution - -To avoid unwanted charges to your AWS account, delete all the AWS resources created during this deployment -:::