diff --git a/schedulers/terraform/self-managed-airflow/README.md b/schedulers/terraform/self-managed-airflow/README.md index eec59b63d..d6382c6e7 100644 --- a/schedulers/terraform/self-managed-airflow/README.md +++ b/schedulers/terraform/self-managed-airflow/README.md @@ -33,7 +33,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | [airflow\_s3\_bucket](#module\_airflow\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | | [amp\_ingest\_irsa](#module\_amp\_ingest\_irsa) | aws-ia/eks-blueprints-addon/aws | ~> 1.0 | | [db](#module\_db) | terraform-aws-modules/rds/aws | ~> 5.0 | -| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | +| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.34 | | [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | | [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | | [eks\_data\_addons](#module\_eks\_data\_addons) | aws-ia/eks-data-addons/aws | ~> 1.2.9 | @@ -52,6 +52,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | [aws_iam_policy.airflow_scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.airflow_webserver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.airflow_worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.fluentbit](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.spark](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_prometheus_workspace.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource | @@ -87,6 +88,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.airflow_s3_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.fluent_bit](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.spark_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -98,7 +100,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [db\_private\_subnets](#input\_db\_private\_subnets) | Private Subnets CIDRs. 254 IPs per Subnet/AZ for Airflow DB. | `list(string)` | 
[
  "10.0.20.0/26",
  "10.0.21.0/26"
]
| no | -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.26"` | no | +| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.29"` | no | | [eks\_data\_plane\_subnet\_secondary\_cidr](#input\_eks\_data\_plane\_subnet\_secondary\_cidr) | Secondary CIDR blocks. 32766 IPs per Subnet per Subnet/AZ for EKS Node and Pods | `list(string)` | 
[
  "100.64.0.0/17",
  "100.64.128.0/17"
]
| no | | [enable\_airflow](#input\_enable\_airflow) | Enable Apache Airflow | `bool` | `true` | no | | [enable\_airflow\_spark\_example](#input\_enable\_airflow\_spark\_example) | Enable Apache Airflow and Spark Operator example | `bool` | `false` | no | diff --git a/schedulers/terraform/self-managed-airflow/addons.tf b/schedulers/terraform/self-managed-airflow/addons.tf index d8d3361d3..e2f7001cf 100644 --- a/schedulers/terraform/self-managed-airflow/addons.tf +++ b/schedulers/terraform/self-managed-airflow/addons.tf @@ -3,7 +3,7 @@ #--------------------------------------------------------------- module "ebs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 5.20" + version = "~> 5.34" role_name_prefix = format("%s-%s-", local.name, "ebs-csi-driver") attach_ebs_csi_policy = true oidc_providers = { @@ -117,16 +117,16 @@ module "eks_blueprints_addons" { #--------------------------------------- enable_aws_for_fluentbit = true aws_for_fluentbit_cw_log_group = { + create = true use_name_prefix = false name = "/${local.name}/aws-fluentbit-logs" # Add-on creates this log group retention_in_days = 30 } - # Additional IRSA policies for FluentBit add-on to access AWS services(e.g., CW Logs, S3 etc.) aws_for_fluentbit = { - s3_bucket_arns = [ - module.fluentbit_s3_bucket.s3_bucket_arn, - "${module.fluentbit_s3_bucket.s3_bucket_arn}/*}" - ] + create_namespace = true + namespace = "aws-for-fluentbit" + create_role = true + role_policies = { "policy1" = aws_iam_policy.fluentbit.arn } values = [templatefile("${path.module}/helm-values/aws-for-fluentbit-values.yaml", { region = local.region, cloudwatch_log_group = "/${local.name}/aws-fluentbit-logs" @@ -155,7 +155,7 @@ module "eks_blueprints_addons" { amp_url = "https://aps-workspaces.${local.region}.amazonaws.com/workspaces/${aws_prometheus_workspace.amp[0].id}" }) : templatefile("${path.module}/helm-values/kube-prometheus.yaml", {}) ] - chart_version = "48.1.1" + chart_version = "48.2.3" set_sensitive = [ { name = "grafana.adminPassword" @@ -167,6 +167,7 @@ module "eks_blueprints_addons" { tags = local.tags } + #--------------------------------------------------------------- # Data on EKS Kubernetes Addons #--------------------------------------------------------------- @@ -181,47 +182,23 @@ module "eks_data_addons" { #--------------------------------------------------------------- enable_airflow = true airflow_helm_config = { - airflow_namespace = try(kubernetes_namespace_v1.airflow[0].metadata[0].name, local.airflow_namespace) - + namespace = try(kubernetes_namespace_v1.airflow[0].metadata[0].name, local.airflow_namespace) + version = "1.11.0" values = [templatefile("${path.module}/helm-values/airflow-values.yaml", { # Airflow Postgres RDS Config - airflow_version = local.airflow_version airflow_db_user = local.airflow_name airflow_db_pass = try(sensitive(aws_secretsmanager_secret_version.postgres[0].secret_string), "") airflow_db_name = try(module.db[0].db_instance_name, "") airflow_db_host = try(element(split(":", module.db[0].db_instance_endpoint), 0), "") + #Service Accounts + worker_service_account = try(kubernetes_service_account_v1.airflow_worker[0].metadata[0].name, local.airflow_workers_service_account) + scheduler_service_account = try(kubernetes_service_account_v1.airflow_scheduler[0].metadata[0].name, local.airflow_scheduler_service_account) + webserver_service_account = try(kubernetes_service_account_v1.airflow_webserver[0].metadata[0].name, local.airflow_webserver_service_account) # S3 bucket config for Logs s3_bucket_name = try(module.airflow_s3_bucket[0].s3_bucket_id, "") webserver_secret_name = local.airflow_webserver_secret_name efs_pvc = local.efs_pvc })] - # Use only when Apache Airflow is enabled with `airflow-core.tf` resources - set = var.enable_amazon_prometheus ? [ - { - name = "scheduler.serviceAccount.create" - value = false - }, - { - name = "scheduler.serviceAccount.name" - value = try(kubernetes_service_account_v1.airflow_scheduler[0].metadata[0].name, local.airflow_scheduler_service_account) - }, - { - name = "webserver.serviceAccount.create" - value = false - }, - { - name = "webserver.serviceAccount.name" - value = try(kubernetes_service_account_v1.airflow_webserver[0].metadata[0].name, local.airflow_webserver_service_account) - }, - { - name = "workers.serviceAccount.create" - value = false - }, - { - name = "workers.serviceAccount.name" - value = try(kubernetes_service_account_v1.airflow_worker[0].metadata[0].name, local.airflow_workers_service_account) - } - ] : [] } #--------------------------------------------------------------- @@ -243,6 +220,11 @@ module "eks_data_addons" { EOT ] } + + #--------------------------------------------------------------- + # Enable Karpenter Resources for Spark team A + #--------------------------------------------------------------- + enable_karpenter_resources = true karpenter_resources_helm_config = { spark-compute-optimized = { @@ -353,6 +335,15 @@ resource "aws_secretsmanager_secret_version" "grafana" { secret_string = random_password.grafana.result } +#--------------------------------------------------------------- +# IAM Policy for FluentBit Add-on +#--------------------------------------------------------------- +resource "aws_iam_policy" "fluentbit" { + description = "IAM policy policy for FluentBit" + name = "${local.name}-fluentbit-additional" + policy = data.aws_iam_policy_document.fluent_bit.json +} + #--------------------------------------------------------------- # S3 log bucket for FluentBit #--------------------------------------------------------------- @@ -374,3 +365,24 @@ module "fluentbit_s3_bucket" { tags = local.tags } + +#--------------------------------------------------------------- +# IAM policy for FluentBit +#--------------------------------------------------------------- +data "aws_iam_policy_document" "fluent_bit" { + statement { + sid = "" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:s3:::${module.fluentbit_s3_bucket.s3_bucket_id}/*"] + + actions = [ + "s3:ListBucket", + "s3:PutObject", + "s3:PutObjectAcl", + "s3:GetObject", + "s3:GetObjectAcl", + "s3:DeleteObject", + "s3:DeleteObjectVersion" + ] + } +} diff --git a/schedulers/terraform/self-managed-airflow/airflow-core.tf b/schedulers/terraform/self-managed-airflow/airflow-core.tf index 26ab1495f..396b092e8 100644 --- a/schedulers/terraform/self-managed-airflow/airflow-core.tf +++ b/schedulers/terraform/self-managed-airflow/airflow-core.tf @@ -8,15 +8,15 @@ module "db" { identifier = local.airflow_name + # All available versions: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_PostgreSQL.html#PostgreSQL.Concepts engine = "postgres" - engine_version = "14.10" - family = "postgres14" - major_engine_version = "14" - instance_class = "db.m6i.xlarge" + engine_version = "14" + family = "postgres14" # DB parameter group + major_engine_version = "14" # DB option group + instance_class = "db.t4g.large" - storage_type = "io1" - allocated_storage = 100 - iops = 3000 + allocated_storage = 20 + max_allocated_storage = 100 db_name = local.airflow_name username = local.airflow_name @@ -33,7 +33,7 @@ module "db" { enabled_cloudwatch_logs_exports = ["postgresql", "upgrade"] create_cloudwatch_log_group = true - backup_retention_period = 5 + backup_retention_period = 1 skip_final_snapshot = true deletion_protection = false diff --git a/schedulers/terraform/self-managed-airflow/helm-values/airflow-values.yaml b/schedulers/terraform/self-managed-airflow/helm-values/airflow-values.yaml index 40f6bf0c8..c3b3c87a0 100644 --- a/schedulers/terraform/self-managed-airflow/helm-values/airflow-values.yaml +++ b/schedulers/terraform/self-managed-airflow/helm-values/airflow-values.yaml @@ -6,25 +6,6 @@ securityContext: fsGroup: 65534 -# Airflow home directory -# Used for mount paths -airflowHome: /opt/airflow - -# Default airflow repository -- overridden by all the specific images below -defaultAirflowRepository: apache/airflow - -# Default airflow tag to deploy -defaultAirflowTag: ${airflow_version} - -# Airflow version (Used to make some decisions based on Airflow Version being deployed) -airflowVersion: ${airflow_version} - -################################### -# Images -################################### -images: - migrationsWaitTimeout: 300 - ################################### # Ingress configuration ################################### @@ -54,36 +35,6 @@ ingress: # The Ingress Class for the web Ingress (used only with Kubernetes v1.19 and above) ingressClassName: "alb" -################################### -# `airflow_local_settings` file as a string (can be templated). -################################### -airflowLocalSettings: |- - {{- if semverCompare ">=2.2.0" .Values.airflowVersion }} - {{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }} - from airflow.www.utils import UIAlert - - DASHBOARD_UIALERTS = [ - UIAlert( - 'Usage of a dynamic webserver secret key detected. We recommend a static webserver secret key instead.' - ' See the ' - 'Helm Chart Production Guide for more details.', - category="warning", - roles=["Admin"], - html=True, - ) - ] - {{- end }} - {{- end }} - -################################### -# Enable RBAC (default on most clusters these days) -################################### -rbac: - # Specifies whether RBAC resources should be created - create: true - createSCCRoleBinding: false - ################################### # Airflow executor ################################### @@ -107,118 +58,24 @@ data: ################################### # Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg ################################### -#webserverSecretKey: ~ webserverSecretKeySecretName: ${webserver_secret_name} ################################### # Airflow Worker Config ################################### workers: - # Number of airflow celery workers in StatefulSet - replicas: 1 - # Max number of old replicasets to retain - revisionHistoryLimit: ~ - - # Command to use when running Airflow workers (templated). - command: ~ - # Args to use when running Airflow workers (templated). - args: - - "bash" - - "-c" - - # If the worker stops responding for 5 minutes (5*60s) kill the - # worker and let Kubernetes restart it - livenessProbe: - enabled: true - initialDelaySeconds: 10 - timeoutSeconds: 20 - failureThreshold: 5 - periodSeconds: 60 - - # Update Strategy when worker is deployed as a Deployment - strategy: - rollingUpdate: - maxSurge: "100%" - maxUnavailable: "50%" - - # Allow KEDA autoscaling. - # Persistence.enabled must be set to false to use KEDA. - #keda: - # enabled: false - # namespaceLabels: {} - - # How often KEDA polls the airflow DB to report new scale requests to the HPA - # pollingInterval: 5 - - # How many seconds KEDA will wait before scaling to zero. - # Note that HPA has a separate cooldown period for scale-downs - # cooldownPeriod: 30 - - # Minimum number of workers created by keda - # minReplicaCount: 0 - - # Maximum number of workers created by keda - # maxReplicaCount: 10 - - # Specify HPA related options - # advanced: {} - # horizontalPodAutoscalerConfig: - # behavior: - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Percent - # value: 100 - # periodSeconds: 15 - persistence: # Enable persistent volumes enabled: false - - #resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - # Grace period for tasks to finish after SIGTERM is sent from kubernetes - terminationGracePeriodSeconds: 600 - - # This setting tells kubernetes that its ok to evict - # when it wants to scale a node down. - safeToEvict: true - - # Select certain nodes for airflow worker pods. -# nodeSelector: {} -# priorityClassName: ~ -# affinity: {} - # default worker affinity is: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - podAffinityTerm: - # labelSelector: - # matchLabels: - # component: worker - # topologyKey: kubernetes.io/hostname - # weight: 100 -# tolerations: [] -# topologySpreadConstraints: [] + # Create ServiceAccount + serviceAccount: + create: false + name: ${worker_service_account} ################################### # Airflow scheduler settings ################################### scheduler: - - # If the scheduler stops heartbeating for 5 minutes (5*60s) kill the - # scheduler and let Kubernetes restart it - livenessProbe: - initialDelaySeconds: 10 - timeoutSeconds: 20 - failureThreshold: 5 - periodSeconds: 60 - # Airflow 2.0 allows users to run multiple schedulers, # However this feature is only recommended for MySQL 8+ and Postgres replicas: 2 @@ -231,7 +88,6 @@ scheduler: config: # minAvailable and maxUnavailable are mutually exclusive maxUnavailable: 1 - #minAvailable: 1 resources: limits: @@ -244,96 +100,20 @@ scheduler: # This setting tells kubernetes that its ok to evict # when it wants to scale a node down. safeToEvict: false - - - # Select certain nodes for airflow scheduler pods. -# nodeSelector: {} -# affinity: {} - # default scheduler affinity is: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - podAffinityTerm: - # labelSelector: - # matchLabels: - # component: scheduler - # topologyKey: kubernetes.io/hostname - # weight: 100 -# tolerations: [] -# topologySpreadConstraints: [] + # Create ServiceAccount + serviceAccount: + create: false + name: ${scheduler_service_account} #extraVolumes is required for DAG GitSync #https://github.com/apache/airflow/issues/27476 extraVolumes: - name: git-sync-ssh-key secret: secretName: airflow-ssh-secret - -################################### -# Airflow create user job settings -################################### -createUserJob: - # Limit the lifetime of the job object after it finished execution. - ttlSecondsAfterFinished: 300 - # Command to use when running the create user job (templated). - args: - - "bash" - - "-c" - # The format below is necessary to get `helm lint` happy - - |- - exec \ - airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "users create" "create_user" }} "$@" - - -- - - "-r" - - "{{ .Values.webserver.defaultUser.role }}" - - "-u" - - "{{ .Values.webserver.defaultUser.username }}" - - "-e" - - "{{ .Values.webserver.defaultUser.email }}" - - "-f" - - "{{ .Values.webserver.defaultUser.firstName }}" - - "-l" - - "{{ .Values.webserver.defaultUser.lastName }}" - - "-p" - - "{{ .Values.webserver.defaultUser.password }}" - - -################################### -# Airflow database migration job settings -################################### -migrateDatabaseJob: - enabled: true - # Limit the lifetime of the job object after it finished execution. - ttlSecondsAfterFinished: 300 - # Command to use when running the migrate database job (templated). - command: ~ - # Args to use when running the migrate database job (templated). - args: - - "bash" - - "-c" - # The format below is necessary to get `helm lint` happy - - |- - exec \ - airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb" }} - - ################################### # Airflow webserver settings ################################### webserver: - allowPodLogReading: true - livenessProbe: - initialDelaySeconds: 15 - timeoutSeconds: 30 - failureThreshold: 20 - periodSeconds: 10 - scheme: HTTP - - readinessProbe: - initialDelaySeconds: 15 - timeoutSeconds: 30 - failureThreshold: 20 - periodSeconds: 10 - scheme: HTTP - # Number of webservers replicas: 2 @@ -367,53 +147,11 @@ webserver: service: #type: ClusterIP type: NodePort - ## service annotations - #annotations: {} - ports: - - name: airflow-ui - port: "{{ .Values.ports.airflowUI }}" - - # Select certain nodes for airflow webserver pods. - #nodeSelector: {} - #priorityClassName: ~ - #affinity: {} - # default webserver affinity is: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - podAffinityTerm: - # labelSelector: - # matchLabels: - # component: webserver - # topologyKey: kubernetes.io/hostname - # weight: 100 - #tolerations: [] - #topologySpreadConstraints: [] - -################################### -# Airflow Triggerer Config -################################### -triggerer: - enabled: true - -################################### -# Airflow Dag Processor Config -################################### -dagProcessor: - enabled: false - -################################### -# StatsD settings -################################### -statsd: - enabled: true - resources: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 200m - memory: 256Mi + # Create ServiceAccount + serviceAccount: + create: false + name: ${webserver_service_account} ################################### # PgBouncer settings @@ -421,34 +159,10 @@ statsd: pgbouncer: # Enable PgBouncer enabled: true - # Number of PgBouncer replicas to run in Deployment - replicas: 1 - auth_type: scram-sha-256 - -################################### -# All ports used by chart -################################### -ports: - airflowUI: 8080 - workerLogs: 8793 - triggererLogs: 8794 - statsdIngest: 9125 - statsdScrape: 9102 - pgbouncer: 6543 - pgbouncerScrape: 9127 ################################### # Config settings to go into the mounted airflow.cfg ################################### -# -# Please note that these values are passed through the `tpl` function, so are -# all subject to being rendered as go templates. If you need to include a -# literal `{{` in a value, it must be expressed like this: -# -# a: '{{ "{{ not a template }}" }}' -# -# Do not set config containing secrets via plain text values, use Env Var or k8s secret object -# yamllint disable rule:line-length config: core: dags_folder: '{{ include "airflow_dags" . }}' @@ -466,41 +180,6 @@ config: remote_log_conn_id: 'aws_s3_conn' delete_worker_pods: 'False' encrypt_s3_logs: 'True' - metrics: - statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' - statsd_port: 9125 - statsd_prefix: airflow - statsd_host: '{{ printf "%s-statsd" .Release.Name }}' - webserver: - enable_proxy_fix: 'True' - # For Airflow 1.10 - rbac: 'True' - scheduler: - standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}' - # statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0 - statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' - statsd_port: 9125 - statsd_prefix: airflow - statsd_host: '{{ printf "%s-statsd" .Release.Name }}' - # `run_duration` included for Airflow 1.10 backward compatibility; removed in 2.0. - run_duration: 41460 - kubernetes: - namespace: '{{ .Release.Namespace }}' - # The following `airflow_` entries are for Airflow 1, and can be removed when it is no longer supported. - airflow_configmap: '{{ include "airflow_config" . }}' - airflow_local_settings_configmap: '{{ include "airflow_config" . }}' - pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml' - worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' - worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' - multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}' - # The `kubernetes_executor` section duplicates the `kubernetes` section in Airflow >= 2.5.0 due to an airflow.cfg schema change. - kubernetes_executor: - namespace: '{{ .Release.Namespace }}' - pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml' - worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' - worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' - multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}' -# yamllint enable rule:line-length ################################### @@ -523,8 +202,8 @@ dags: enabled: true # git repo clone url - #repo: git@github.com:Hyper-Mesh/airflow-dags.git - repo: git@github.com:jagpk/sample-airflow-dags.git + repo: git@github.com:Hyper-Mesh/airflow-dags.git + #repo: git@github.com:jagpk/sample-airflow-dags.git branch: main rev: HEAD depth: 1 @@ -548,18 +227,6 @@ dags: github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= - # interval between git sync attempts in seconds - # high values are more likely to cause DAGs to become out of sync between different components - # low values cause more traffic to the remote git repository - wait: 5 - containerName: git-sync - resources: - limits: - cpu: 100m - memory: 512Mi - requests: - cpu: 100m - memory: 512Mi ################################### # Airflow Extra Secrets diff --git a/schedulers/terraform/self-managed-airflow/helm-values/aws-for-fluentbit-values.yaml b/schedulers/terraform/self-managed-airflow/helm-values/aws-for-fluentbit-values.yaml index 0bea5188d..7219002bb 100644 --- a/schedulers/terraform/self-managed-airflow/helm-values/aws-for-fluentbit-values.yaml +++ b/schedulers/terraform/self-managed-airflow/helm-values/aws-for-fluentbit-values.yaml @@ -53,7 +53,7 @@ filter: # cloudWatch: # enabled: false -# This is a new high performance C Plugin for CloudWatchLogs. See docs here https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch +# This is a new high performance C Plugin for CloudWatchLogs. See docs here https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch cloudWatchLogs: enabled: true match: "systempods.*" diff --git a/schedulers/terraform/self-managed-airflow/main.tf b/schedulers/terraform/self-managed-airflow/main.tf index 6a85e38ad..f703e5ee8 100644 --- a/schedulers/terraform/self-managed-airflow/main.tf +++ b/schedulers/terraform/self-managed-airflow/main.tf @@ -53,7 +53,6 @@ locals { azs = slice(data.aws_availability_zones.available.names, 0, 2) airflow_name = "airflow" - airflow_version = "2.5.3" airflow_namespace = "airflow" airflow_scheduler_service_account = "airflow-scheduler" airflow_webserver_service_account = "airflow-webserver" diff --git a/schedulers/terraform/self-managed-airflow/variables.tf b/schedulers/terraform/self-managed-airflow/variables.tf index 2c92f6d41..c2068ae96 100644 --- a/schedulers/terraform/self-managed-airflow/variables.tf +++ b/schedulers/terraform/self-managed-airflow/variables.tf @@ -12,7 +12,7 @@ variable "region" { variable "eks_cluster_version" { description = "EKS Cluster version" - default = "1.26" + default = "1.29" type = string } diff --git a/streaming/flink/karpenter-provisioners/flink-compute-optimized-provisioner.yaml b/streaming/flink/karpenter-provisioners/flink-compute-optimized-provisioner.yaml index 955b95516..e664a603f 100644 --- a/streaming/flink/karpenter-provisioners/flink-compute-optimized-provisioner.yaml +++ b/streaming/flink/karpenter-provisioners/flink-compute-optimized-provisioner.yaml @@ -108,4 +108,4 @@ spec: --BOUNDARY-- tags: - InstanceType: "flink-compute-optimized" # optional, add tags for your own use \ No newline at end of file + InstanceType: "flink-compute-optimized" # optional, add tags for your own use diff --git a/streaming/flink/main.tf b/streaming/flink/main.tf index c382a7f30..17633e082 100755 --- a/streaming/flink/main.tf +++ b/streaming/flink/main.tf @@ -4,8 +4,6 @@ locals { vpc_cidr = var.vpc_cidr azs = slice(data.aws_availability_zones.available.names, 0, 2) - karpenter_iam_role_name = format("%s-%s", "karpenter", local.name) - account_id = data.aws_caller_identity.current.account_id partition = data.aws_partition.current.partition