From 7430c6e08252c09919429f6d9fa8b0af674c2c48 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Sat, 9 Nov 2024 11:46:57 +0200 Subject: [PATCH 1/5] Update templates --- config/clusters/templates/common/support.values.yaml | 8 ++++++++ terraform/aws/projects/strudel.tfvars | 1 + 2 files changed, 9 insertions(+) diff --git a/config/clusters/templates/common/support.values.yaml b/config/clusters/templates/common/support.values.yaml index 431149bde4..eaa6ae4bfa 100644 --- a/config/clusters/templates/common/support.values.yaml +++ b/config/clusters/templates/common/support.values.yaml @@ -27,6 +27,14 @@ grafana: hosts: - grafana.{{ cluster_name }}.2i2c.cloud {% if provider == "aws" %} +aws-ce-grafana-backend: + enabled: true + envBasedConfig: + clusterName: {{ cluster_name }} + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: {{ output of `terraform output -raw aws_ce_grafana_backend_k8s_sa_annotation` }} + cluster-autoscaler: enabled: true autoDiscovery: diff --git a/terraform/aws/projects/strudel.tfvars b/terraform/aws/projects/strudel.tfvars index 8cd8483e23..11812c4c61 100644 --- a/terraform/aws/projects/strudel.tfvars +++ b/terraform/aws/projects/strudel.tfvars @@ -15,6 +15,7 @@ enable_aws_ce_grafana_backend_iam = true #user_buckets = { # "scratch-staging" : { # "delete_after" : 7, +# "tags" : { "2i2c:hub-name" : "staging" }, # }, # # Tip: add more scratch buckets below, if this cluster will be multi-tenant #} From 2fc37bdc4406dd98e0da861bc9f58a3bbd4c2b77 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Mon, 11 Nov 2024 14:40:25 +0200 Subject: [PATCH 2/5] Update the command --- config/clusters/templates/common/support.values.yaml | 2 +- deployer/commands/generate/dedicated_cluster/aws.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/clusters/templates/common/support.values.yaml b/config/clusters/templates/common/support.values.yaml index eaa6ae4bfa..d06c809277 100644 --- a/config/clusters/templates/common/support.values.yaml +++ b/config/clusters/templates/common/support.values.yaml @@ -33,7 +33,7 @@ aws-ce-grafana-backend: clusterName: {{ cluster_name }} serviceAccount: annotations: - eks.amazonaws.com/role-arn: {{ output of `terraform output -raw aws_ce_grafana_backend_k8s_sa_annotation` }} + eks.amazonaws.com/role-arn: {{ aws_grafana_sa_annotation }} cluster-autoscaler: enabled: true diff --git a/deployer/commands/generate/dedicated_cluster/aws.py b/deployer/commands/generate/dedicated_cluster/aws.py index 6aa86bb12c..85195d954a 100644 --- a/deployer/commands/generate/dedicated_cluster/aws.py +++ b/deployer/commands/generate/dedicated_cluster/aws.py @@ -110,6 +110,10 @@ def aws( ..., prompt="The AWS account id or alias. Declare 2i2c for 2i2c's SSO based accounts and paid_by_us=true", ), + aws_ce_grafana_backend_k8s_sa_annotation: str = typer.Option( + ..., + help="Output of `terraform output -raw aws_ce_grafana_backend_k8s_sa_annotation` for the cluster", + ), force: bool = typer.Option( False, "--force", @@ -139,6 +143,7 @@ def aws( "cluster_region": cluster_region, "sign_in_url": sign_in_url, "paid_by_us": str(paid_by_us).lower(), + "aws_grafana_sa_annotation": aws_ce_grafana_backend_k8s_sa_annotation, } if not check_before_continuing_with_generate_command( From 3d6625f16eeda79d6bb05973f035c8e1d45f8045 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Mon, 11 Nov 2024 14:51:10 +0200 Subject: [PATCH 3/5] Update docs --- docs/howto/cost-attribution/aws.md | 65 +++++++++++++----------------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/docs/howto/cost-attribution/aws.md b/docs/howto/cost-attribution/aws.md index 33ff0d6207..18fe77fb2b 100644 --- a/docs/howto/cost-attribution/aws.md +++ b/docs/howto/cost-attribution/aws.md @@ -22,6 +22,21 @@ attribute cost to. - `alpha.eksctl.io/cluster-name` - `kubernetes.io/cluster/` +```{important} +Currently, on clusters that have a k8s version greater or equal with 1.30, +terraform managed resources already have the `2i2c.org/cluster-name` +tag configured via the `default_tags` variable, and eksctl managed resources +already have the tag configured for node groups via `nodegroup.libsonnet`. + +On clusters that have a k8s version less than 1.30, eksctl managed resources, +the `alpha.eksctl.io/cluster-name` and `kubernetes.io/cluster/` +tags are present and used instead. + +New clusters have _all_ eksctl managed resources configured to be tagged, not +just the node groups. This isn't important to ensure for existing clusters' +cost attribution though. +``` + The system also relies on the tag `2i2c:hub-name` to be specified in addition to the tags above for any cloud infra tied to specific hubs. @@ -31,21 +46,7 @@ create cloud resources to represent k8s resources (block storage volumes for k8s PV resources referencing certain storage classes, and load balancers for k8s Service's of type LoadBalancer). -1. _Configure `2i2c.org/cluster-name` tags_ - - No configuration is needed. - - ```{note} - Terraform managed resources already have the tag configured via the - `default_tags` variable, and eksctl managed resources already have the tag - configured for node groups via `nodegroup.libsonnet`. - - New clusters have _all_ eksctl managed resources configured to be tagged, not - just the node groups. This isn't important to ensure for existing clusters' - cost attribution though. - ``` - -2. _Configure `2i2c:hub-name` tags_ +1. _Configure `2i2c:hub-name` tags_ For any resource _specific to a hub_, declare an additional tag `2i2c:hub-name=`. If this isn't done, they will be listed under a @@ -54,16 +55,16 @@ Service's of type LoadBalancer). The following resources are known to be hub specific in some cases and known to incur costs. - - S3 buckets in terraform - - EFS storage in terraform - - EBS volumes in terraform - - Node groups in eksctl + - **S3 buckets** in terraform + - **EFS storage** in terraform + - **EBS volumes** in terraform + - **Node groups** in eksctl ```{important} If EFS, EBS or nodegroups are not split based on the hub they're deployed to - and instead they are shared by the entire cluster, then splliting it in order - to add the `2i2c:hub-name` tag will be an opt-in feature because it would - incur additional cloud costs and startup times for communities. + and instead they are shared by the entire cluster, then splitting it in order + to add the `2i2c:hub-name` tag is an opt-in feature because the split incurs + additional cloud costs and startup times for communities. See the following GitHub issue for additional context https://github.com/2i2c-org/infrastructure/issues/4928#issuecomment-2417091407 @@ -72,29 +73,17 @@ Service's of type LoadBalancer). Search and mimic configuration of other clusters to understand how to configure the `2i2c:hub-name` tags for specific cloud infra types. -3. _Apply changes_ +2. _Apply changes_ 1. If you changed anything in terraform, apply those changes. 2. If you changed anything in eksctl, apply those changed by re-creating those resources. - 3. If the eksctl cluster is listed and unchecked in this [github reference - issue], and versioned older than k8s 1.29 or older, it needs to have its - node groups re-created to get the implicitly configured - `2i2c.org/cluster-name` tag unless you've not already just done this to - apply a `2i2c:hub-name` tag. - - Reference our [documentation on doing node group - upgrades](upgrade-cluster:aws:node-groups) for details. - 4. Update the [github reference issue] and ensure the checkbox is ticked for - this cluster. - - [github reference issue]: https://github.com/2i2c-org/infrastructure/issues/4885 ### 2. Enable cost allocation tags Enabling cost allocation tags via terraform can be done for standalone AWS -accounts, but not for member accounts part of an organization. Due to this, -we'll provide separate ways of doing this depending on the situation. +accounts, but not for member accounts part of an organization that we don't manage. +Due to this, we'll provide separate ways of doing this depending on the situation. `````{tab-set} @@ -194,7 +183,7 @@ storage disks dynamically provisioned in case that's relevant in the future. You can optionally backfill billing data to tags having been around for a while but not enabled as cost allocation tags. -You can do request this to be done once a day, and it takes a several hours to +You can request this to be done once a day, and it takes a several hours to process the request. Make a request through the AWS web console by navigating to "Cost allocation tags" under "Billing and Cost Management", then from there click the "Backfill tags" button. From 14a059250f1b8c6b08855f2f9bde33faaf6d437e Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Tue, 12 Nov 2024 10:08:09 +0200 Subject: [PATCH 4/5] Add the tags to the correct tfvars --- terraform/aws/projects/strudel.tfvars | 1 - terraform/aws/projects/template.tfvars | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/terraform/aws/projects/strudel.tfvars b/terraform/aws/projects/strudel.tfvars index 11812c4c61..8cd8483e23 100644 --- a/terraform/aws/projects/strudel.tfvars +++ b/terraform/aws/projects/strudel.tfvars @@ -15,7 +15,6 @@ enable_aws_ce_grafana_backend_iam = true #user_buckets = { # "scratch-staging" : { # "delete_after" : 7, -# "tags" : { "2i2c:hub-name" : "staging" }, # }, # # Tip: add more scratch buckets below, if this cluster will be multi-tenant #} diff --git a/terraform/aws/projects/template.tfvars b/terraform/aws/projects/template.tfvars index 2852094bc9..028548f4ac 100644 --- a/terraform/aws/projects/template.tfvars +++ b/terraform/aws/projects/template.tfvars @@ -7,12 +7,15 @@ region = "{{ cluster_region }}" cluster_name = "{{ cluster_name }}" cluster_nodes_location = "{{ cluster_region }}a" +enable_aws_ce_grafana_backend_iam = true + # Tip: uncomment and fill the missing info in the lines below if you want # to setup scratch buckets for the hubs on this cluster. # #user_buckets = { # "scratch-staging" : { # "delete_after" : 7, +# "tags" : { "2i2c:hub-name" : "staging" }, # }, # # Tip: add more scratch buckets below, if this cluster will be multi-tenant #} @@ -28,3 +31,4 @@ cluster_nodes_location = "{{ cluster_region }}a" # }, # # Tip: add more namespaces below, if this cluster will be multi-tenant #} + From fc8a5641033965d2a22502fd43662a61f97adc76 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Tue, 12 Nov 2024 11:56:08 +0200 Subject: [PATCH 5/5] The annotation has to be set manually due to the order of file generation --- .../templates/common/support.values.yaml | 2 +- .../generate/dedicated_cluster/aws.py | 5 --- .../deploy-support/configure-support.md | 38 +++++++++++-------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/config/clusters/templates/common/support.values.yaml b/config/clusters/templates/common/support.values.yaml index d06c809277..b27cab4d8d 100644 --- a/config/clusters/templates/common/support.values.yaml +++ b/config/clusters/templates/common/support.values.yaml @@ -33,7 +33,7 @@ aws-ce-grafana-backend: clusterName: {{ cluster_name }} serviceAccount: annotations: - eks.amazonaws.com/role-arn: {{ aws_grafana_sa_annotation }} + eks.amazonaws.com/role-arn: cluster-autoscaler: enabled: true diff --git a/deployer/commands/generate/dedicated_cluster/aws.py b/deployer/commands/generate/dedicated_cluster/aws.py index 85195d954a..6aa86bb12c 100644 --- a/deployer/commands/generate/dedicated_cluster/aws.py +++ b/deployer/commands/generate/dedicated_cluster/aws.py @@ -110,10 +110,6 @@ def aws( ..., prompt="The AWS account id or alias. Declare 2i2c for 2i2c's SSO based accounts and paid_by_us=true", ), - aws_ce_grafana_backend_k8s_sa_annotation: str = typer.Option( - ..., - help="Output of `terraform output -raw aws_ce_grafana_backend_k8s_sa_annotation` for the cluster", - ), force: bool = typer.Option( False, "--force", @@ -143,7 +139,6 @@ def aws( "cluster_region": cluster_region, "sign_in_url": sign_in_url, "paid_by_us": str(paid_by_us).lower(), - "aws_grafana_sa_annotation": aws_ce_grafana_backend_k8s_sa_annotation, } if not check_before_continuing_with_generate_command( diff --git a/docs/hub-deployment-guide/deploy-support/configure-support.md b/docs/hub-deployment-guide/deploy-support/configure-support.md index fd31021a8e..cf6fd71008 100644 --- a/docs/hub-deployment-guide/deploy-support/configure-support.md +++ b/docs/hub-deployment-guide/deploy-support/configure-support.md @@ -16,26 +16,32 @@ need to recreate them, only update them if required. In the `infrastructure` repo, the full filepath should be: `config/clusters//support.values.yaml`. -Checkout the template support values file in `config/clusters/templates/common/support.values.yaml` for an example configuration. If the cluster is running on GCP or AWS, the deployer should have been generated this file already. +If the cluster is running on GCP or AWS, the deployer should have been generated this file already. -If you are deploying the support chart on an Azure cluster, you **must** manually create such a file using the template mentioned above. Also, you must set an annotation for `ingress-nginx`'s k8s Service resource by including the following in your `support.values.yaml` file: +1. If you are deploying the support chart on an AWS cluster, you **must** also manually update the `aws-ce-grafana-backend` service account annotation in the `support.values.yaml` with the output of thew following command: -```yaml -ingress-nginx: - controller: - service: - annotations: - # This annotation is a requirement for use in Azure provided - # LoadBalancer. - # - # ref: https://learn.microsoft.com/en-us/azure/aks/ingress-basic?tabs=azure-cli#basic-configuration - # ref: https://github.com/Azure/AKS/blob/master/CHANGELOG.md#release-2022-09-11 - # ref: https://github.com/Azure/AKS/issues/2907#issuecomment-1109759262 - # ref: https://github.com/kubernetes/ingress-nginx/issues/8501#issuecomment-1108428615 - # - service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path: /healthz +```bash +terraform output -raw aws_ce_grafana_backend_k8s_sa_annotation ``` +2. If you are deploying the support chart on an Azure cluster, you **must** manually create such a file using the template at `config/clusters/templates/common/support.values.yaml`. Also, you must set an annotation for `ingress-nginx`'s k8s Service resource by including the following in your `support.values.yaml` file: + + ```yaml + ingress-nginx: + controller: + service: + annotations: + # This annotation is a requirement for use in Azure provided + # LoadBalancer. + # + # ref: https://learn.microsoft.com/en-us/azure/aks/ingress-basic?tabs=azure-cli#basic-configuration + # ref: https://github.com/Azure/AKS/blob/master/CHANGELOG.md#release-2022-09-11 + # ref: https://github.com/Azure/AKS/issues/2907#issuecomment-1109759262 + # ref: https://github.com/kubernetes/ingress-nginx/issues/8501#issuecomment-1108428615 + # + service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path: /healthz + ``` + ## Edit your `cluster.yaml` file Add the following config as a top-level key to your `cluster.yaml` file.