Skip to content

Commit 13503e1

Browse files
ratnopamcratnopamc
authored
fix: Flink operator blueprint deployment errors (#731)
1 parent 41d6899 commit 13503e1

File tree

5 files changed

+36
-31
lines changed

5 files changed

+36
-31
lines changed

‎analytics/terraform/spark-k8s-operator/examples/s3-tables/s3table-spark-operator.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ spec:
2323
imagePullPolicy: IfNotPresent
2424
mainApplicationFile: "s3a://<S3_BUCKET>/s3table-example/scripts/s3table-iceberg-pyspark.py"
2525
arguments:
26-
- "s3a://<S3_BUCKET>/s3table-example/input/"
27-
- "<S3TABLE_ARN>"
26+
- "s3a://<S3_BUCKET>/s3table-example/input/"
27+
- "<S3TABLE_ARN>"
2828
sparkConf:
2929
"spark.app.name": "s3table-example"
3030
"spark.kubernetes.driver.pod.name": "s3table-example"
@@ -39,7 +39,7 @@ spec:
3939
"spark.hadoop.fs.s3a.readahead.range": "256K"
4040
"spark.hadoop.fs.s3a.input.fadvise": "random"
4141
"spark.hadoop.fs.s3a.aws.credentials.provider.mapping": "com.amazonaws.auth.WebIdentityTokenCredentialsProvider=software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider"
42-
"spark.hadoop.fs.s3a.aws.credentials.provider": "software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider" # AWS SDK V2 https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
42+
"spark.hadoop.fs.s3a.aws.credentials.provider": "software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider" # AWS SDK V2 https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
4343

4444
restartPolicy:
4545
type: OnFailure

‎streaming/flink/README.md

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,21 @@
66
| Name | Version |
77
|------|---------|
88
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
9-
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72 |
9+
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.34 |
1010
| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.4.1 |
11-
| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14 |
11+
| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 2.0 |
1212
| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
13-
| <a name="requirement_random"></a> [random](#requirement\_random) | 3.3.2 |
13+
| <a name="requirement_random"></a> [random](#requirement\_random) | 3.6.0 |
1414

1515
## Providers
1616

1717
| Name | Version |
1818
|------|---------|
19-
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72 |
20-
| <a name="provider_aws.ecr"></a> [aws.ecr](#provider\_aws.ecr) | >= 3.72 |
21-
| <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | >= 1.14 |
19+
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.34 |
20+
| <a name="provider_aws.ecr"></a> [aws.ecr](#provider\_aws.ecr) | >= 5.34 |
21+
| <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | >= 2.0 |
2222
| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | >= 2.10 |
23-
| <a name="provider_random"></a> [random](#provider\_random) | 3.3.2 |
23+
| <a name="provider_random"></a> [random](#provider\_random) | 3.6.0 |
2424

2525
## Modules
2626

@@ -47,10 +47,10 @@
4747
| [aws_prometheus_workspace.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource |
4848
| [aws_secretsmanager_secret.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource |
4949
| [aws_secretsmanager_secret_version.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource |
50-
| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
50+
| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource |
5151
| [kubernetes_role.flink](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role) | resource |
5252
| [kubernetes_role_binding.flink](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource |
53-
| [random_password.grafana](https://registry.terraform.io/providers/hashicorp/random/3.3.2/docs/resources/password) | resource |
53+
| [random_password.grafana](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/password) | resource |
5454
| [aws_ami.x86](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
5555
| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
5656
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@@ -62,7 +62,7 @@
6262
| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
6363
| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
6464
| [aws_secretsmanager_secret_version.admin_password_version](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret_version) | data source |
65-
| [kubectl_path_documents.karpenter_provisioners](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source |
65+
| [kubectl_path_documents.karpenter_provisioners](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/data-sources/path_documents) | data source |
6666

6767
## Inputs
6868

‎streaming/flink/addons.tf

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ module "eks_blueprints_addons" {
8282
}
8383
}
8484
karpenter = {
85-
chart_version = "v0.33.1"
85+
chart_version = "v0.37.1"
8686
repository_username = data.aws_ecrpublic_authorization_token.token.user_name
8787
repository_password = data.aws_ecrpublic_authorization_token.token.password
8888
}
@@ -99,7 +99,7 @@ module "eks_blueprints_addons" {
9999
#---------------------------------------
100100
# AWS for FluentBit - DaemonSet
101101
#---------------------------------------
102-
enable_aws_for_fluentbit = true
102+
enable_aws_for_fluentbit = false
103103
aws_for_fluentbit_cw_log_group = {
104104
create = true
105105
use_name_prefix = false
@@ -120,19 +120,25 @@ module "eks_blueprints_addons" {
120120
})]
121121
}
122122

123+
#---------------------------------------
124+
# AWS Load Balancer Controller Add-on
125+
#---------------------------------------
123126
enable_aws_load_balancer_controller = true
127+
# turn off the mutating webhook for services because we are using
128+
# service.beta.kubernetes.io/aws-load-balancer-type: external
124129
aws_load_balancer_controller = {
125-
values = [
126-
<<-EOT
127-
clusterName: ${module.eks.cluster_name}
128-
EOT
129-
]
130+
set = [{
131+
name = "enableServiceMutatorWebhook"
132+
value = "false"
133+
}]
130134
}
131135

136+
#---------------------------------------
137+
# Ingress Nginx Add-on
138+
#---------------------------------------
132139
enable_ingress_nginx = true
133140
ingress_nginx = {
134-
timeout = "300"
135-
values = [templatefile("${path.module}/helm-values/nginx-values.yaml", {})]
141+
values = [templatefile("${path.module}/helm-values/nginx-values.yaml", {})]
136142
}
137143

138144
# Required Flink Deployment
@@ -182,18 +188,17 @@ module "eks_blueprints_addons" {
182188
# Data on EKS Kubernetes Addons
183189
#---------------------------------------------------------------
184190
module "eks_data_addons" {
185-
source = "aws-ia/eks-data-addons/aws"
186-
version = "~> 1.0" # ensure to update this to the latest/desired version
187-
191+
source = "aws-ia/eks-data-addons/aws"
192+
version = "~> 1.0" # ensure to update this to the latest/desired version
188193
oidc_provider_arn = module.eks.oidc_provider_arn
189194

190195
#---------------------------------------------------------------
191196
# Flink Operator Add-on
192197
#---------------------------------------------------------------
193198
enable_flink_operator = true
194199
flink_operator_helm_config = {
195-
version = "1.7.0"
196-
values = [templatefile("${path.module}/helm-values/flink-operator-values.yaml", {})]
200+
version = "1.10.0"
201+
values = [templatefile("${path.module}/helm-values/flink-operator-values.yaml", {})]
197202
}
198203

199204
#---------------------------------------------------------------

‎streaming/flink/versions.tf

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ terraform {
44
required_providers {
55
aws = {
66
source = "hashicorp/aws"
7-
version = ">= 3.72"
7+
version = ">= 5.34"
88
}
99
kubernetes = {
1010
source = "hashicorp/kubernetes"
@@ -15,12 +15,12 @@ terraform {
1515
version = ">= 2.4.1"
1616
}
1717
kubectl = {
18-
source = "gavinbunney/kubectl"
19-
version = ">= 1.14"
18+
source = "alekc/kubectl"
19+
version = ">= 2.0"
2020
}
2121
random = {
2222
source = "hashicorp/random"
23-
version = "3.3.2"
23+
version = "3.6.0"
2424
}
2525
}
2626

‎trino

-13 MB
Binary file not shown.

0 commit comments

Comments
 (0)