From ecc872406be9b86bdf65992597ce41958af6b1dd Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Mar 2024 14:08:38 +0100 Subject: [PATCH 1/5] pangeo-hubs: increase prometheus-server memory req/limit from 16 to 24Gi --- config/clusters/pangeo-hubs/support.values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/clusters/pangeo-hubs/support.values.yaml b/config/clusters/pangeo-hubs/support.values.yaml index 354cd47776..7135427f75 100644 --- a/config/clusters/pangeo-hubs/support.values.yaml +++ b/config/clusters/pangeo-hubs/support.values.yaml @@ -35,6 +35,6 @@ prometheus: - prometheus.gcp.pangeo.2i2c.cloud resources: requests: - memory: 16Gi + memory: 24Gi limits: - memory: 16Gi + memory: 24Gi From ea1060ef507e735874d9b200b7321e302227c528 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Mar 2024 15:52:13 +0100 Subject: [PATCH 2/5] new cluster automation: add comment with cloud infra link --- config/clusters/templates/gcp/cluster.yaml | 2 +- docs/hub-deployment-guide/new-cluster/aws.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/clusters/templates/gcp/cluster.yaml b/config/clusters/templates/gcp/cluster.yaml index 65a16ad442..0c7c816e31 100644 --- a/config/clusters/templates/gcp/cluster.yaml +++ b/config/clusters/templates/gcp/cluster.yaml @@ -1,5 +1,5 @@ name: {{ cluster_name }} -provider: gcp +provider: gcp # https://console.cloud.google.com/kubernetes/clusters/details/{{ cluster_region }}/{{ cluster_name }}-cluster/observability?project={{ project_id }} gcp: key: enc-deployer-credentials.secret.json project: {{ project_id }} diff --git a/docs/hub-deployment-guide/new-cluster/aws.md b/docs/hub-deployment-guide/new-cluster/aws.md index 030ff8f7ec..c08ec13dc6 100644 --- a/docs/hub-deployment-guide/new-cluster/aws.md +++ b/docs/hub-deployment-guide/new-cluster/aws.md @@ -250,7 +250,7 @@ have least amount of permissions possible. ```yaml name: - provider: aws + provider: aws # aws: key: enc-deployer-credentials.secret.json clusterType: eks From bca2def8d4643ffff848be66eb68c5d5554067ef Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Mar 2024 15:52:46 +0100 Subject: [PATCH 3/5] deployer: add aws.billing.paid_by_us to cluster.yaml schema --- .../commands/validate/cluster.schema.yaml | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/deployer/commands/validate/cluster.schema.yaml b/deployer/commands/validate/cluster.schema.yaml index 602776363e..ace570eef2 100644 --- a/deployer/commands/validate/cluster.schema.yaml +++ b/deployer/commands/validate/cluster.schema.yaml @@ -66,10 +66,18 @@ properties: description: | Configuration to connect to the cluster on GCP. Is used when provider is set to `gcp` + required: + - key + - project + - cluster + - zone + - billing properties: billing: type: object additionalProperties: false + required: + - paid_by_us # if paid_by_us is true, we *must* provide bigquery information # on where to find the costs export for this project if: @@ -79,15 +87,13 @@ properties: then: required: - bigquery - required: - - paid_by_us description: | Billing information for this cluster properties: paid_by_us: type: boolean description: | - Set to true if the cloud cost is paid by us, and expected to be + Set to true if the cloud cost is paid by 2i2c, and expected to be recovered from actual users. bigquery: type: object @@ -133,15 +139,15 @@ properties: If the cluster is a zonal cluster, this specifies the zone in which the cluster is. If it's a regional cluster, this specifies the region the cluster is in. - required: - - key - - project - - cluster - - zone - - billing aws: type: object additionalProperties: false + required: + - key + - clusterType + - clusterName + - region + - billing description: | Configuration to connect to the cluster on AWS. Is used when provider is set to `aws`. @@ -168,14 +174,24 @@ properties: type: string description: | The AWS region the cluster is in. - required: - - key - - clusterType - - clusterName - - region + billing: + type: object + additionalProperties: false + required: + - paid_by_us + properties: + paid_by_us: + type: boolean + description: | + Set to true if the cloud cost is paid by 2i2c, and expected to be + recovered from actual users. azure: type: object additionalProperties: false + required: + - key + - resource_group + - cluster description: | Configuration to connect to a cluster on Azure. Is used when provider is set to `azure` @@ -197,10 +213,6 @@ properties: type: string description: | Name of the cluster inside this Azure Resource Group - required: - - key - - resource_group - - cluster hubs: type: array description: | From e3ae87eaf3288aa9b11b4f4e0518a7781c7e811c Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Mar 2024 15:53:18 +0100 Subject: [PATCH 4/5] Initialize aws.billing.paid_by_us --- config/clusters/2i2c-aws-us/cluster.yaml | 2 ++ config/clusters/bican/cluster.yaml | 2 ++ config/clusters/catalystproject-africa/cluster.yaml | 2 ++ config/clusters/earthscope/cluster.yaml | 2 ++ config/clusters/gridsst/cluster.yaml | 2 ++ config/clusters/jupyter-meets-the-earth/cluster.yaml | 4 +++- config/clusters/linc/cluster.yaml | 2 ++ config/clusters/nasa-cryo/cluster.yaml | 2 ++ config/clusters/nasa-esdis/cluster.yaml | 2 ++ config/clusters/nasa-ghg/cluster.yaml | 2 ++ config/clusters/nasa-veda/cluster.yaml | 2 ++ config/clusters/openscapes/cluster.yaml | 2 ++ config/clusters/opensci/cluster.yaml | 2 ++ config/clusters/smithsonian/cluster.yaml | 2 ++ config/clusters/ubc-eoas/cluster.yaml | 2 ++ config/clusters/victor/cluster.yaml | 2 ++ 16 files changed, 33 insertions(+), 1 deletion(-) diff --git a/config/clusters/2i2c-aws-us/cluster.yaml b/config/clusters/2i2c-aws-us/cluster.yaml index a1dd7f42bc..0dea4cf1bb 100644 --- a/config/clusters/2i2c-aws-us/cluster.yaml +++ b/config/clusters/2i2c-aws-us/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: 2i2c-aws-us region: us-west-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/bican/cluster.yaml b/config/clusters/bican/cluster.yaml index 8f3055c1db..5a31f2dcc0 100644 --- a/config/clusters/bican/cluster.yaml +++ b/config/clusters/bican/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: bican region: us-east-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/catalystproject-africa/cluster.yaml b/config/clusters/catalystproject-africa/cluster.yaml index 9794d2fe37..4b15e81a22 100644 --- a/config/clusters/catalystproject-africa/cluster.yaml +++ b/config/clusters/catalystproject-africa/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: catalystproject-africa region: af-south-1 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/earthscope/cluster.yaml b/config/clusters/earthscope/cluster.yaml index 0a49671aa2..5b78fd4d21 100644 --- a/config/clusters/earthscope/cluster.yaml +++ b/config/clusters/earthscope/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: earthscope region: us-east-2 + billing: + paid_by_us: false support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/gridsst/cluster.yaml b/config/clusters/gridsst/cluster.yaml index 887b7006b4..11714a1193 100644 --- a/config/clusters/gridsst/cluster.yaml +++ b/config/clusters/gridsst/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: gridsst region: us-west-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/jupyter-meets-the-earth/cluster.yaml b/config/clusters/jupyter-meets-the-earth/cluster.yaml index 6e93a00caf..99cad4e452 100644 --- a/config/clusters/jupyter-meets-the-earth/cluster.yaml +++ b/config/clusters/jupyter-meets-the-earth/cluster.yaml @@ -1,10 +1,12 @@ name: jupyter-meets-the-earth -provider: aws # https://286354552638.signin.aws.amazon.com/console +provider: aws # https://286354552638.signin.aws.amazon.com/console aws: key: enc-deployer-credentials.secret.json clusterType: eks clusterName: jupyter-meets-the-earth region: us-west-2 + billing: + paid_by_us: false support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/linc/cluster.yaml b/config/clusters/linc/cluster.yaml index 539b9289be..5422b26fab 100644 --- a/config/clusters/linc/cluster.yaml +++ b/config/clusters/linc/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: linc region: us-east-1 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/nasa-cryo/cluster.yaml b/config/clusters/nasa-cryo/cluster.yaml index c8369609b6..0a0ba622be 100644 --- a/config/clusters/nasa-cryo/cluster.yaml +++ b/config/clusters/nasa-cryo/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: nasa-cryo region: us-west-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/nasa-esdis/cluster.yaml b/config/clusters/nasa-esdis/cluster.yaml index 98d9b3ea09..6cdfc4df29 100644 --- a/config/clusters/nasa-esdis/cluster.yaml +++ b/config/clusters/nasa-esdis/cluster.yaml @@ -6,6 +6,8 @@ aws: clusterType: eks clusterName: nasa-esdis region: us-west-2 + billing: + paid_by_us: false support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/nasa-ghg/cluster.yaml b/config/clusters/nasa-ghg/cluster.yaml index a93d3978d2..1740a226b9 100644 --- a/config/clusters/nasa-ghg/cluster.yaml +++ b/config/clusters/nasa-ghg/cluster.yaml @@ -6,6 +6,8 @@ aws: clusterType: eks clusterName: nasa-ghg-hub region: us-west-2 + billing: + paid_by_us: false support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/nasa-veda/cluster.yaml b/config/clusters/nasa-veda/cluster.yaml index d503bfba65..7378fdf5bb 100644 --- a/config/clusters/nasa-veda/cluster.yaml +++ b/config/clusters/nasa-veda/cluster.yaml @@ -6,6 +6,8 @@ aws: clusterType: eks clusterName: nasa-veda region: us-west-2 + billing: + paid_by_us: false support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/openscapes/cluster.yaml b/config/clusters/openscapes/cluster.yaml index 6ba7cd23ae..25576f893e 100644 --- a/config/clusters/openscapes/cluster.yaml +++ b/config/clusters/openscapes/cluster.yaml @@ -6,6 +6,8 @@ aws: clusterType: eks clusterName: openscapeshub region: us-west-2 + billing: + paid_by_us: false support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/opensci/cluster.yaml b/config/clusters/opensci/cluster.yaml index 1e5c7b78bc..9a1cc5ff2f 100644 --- a/config/clusters/opensci/cluster.yaml +++ b/config/clusters/opensci/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: opensci region: us-west-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/smithsonian/cluster.yaml b/config/clusters/smithsonian/cluster.yaml index 6b7c0f2307..e61f32331e 100644 --- a/config/clusters/smithsonian/cluster.yaml +++ b/config/clusters/smithsonian/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: smithsonian region: us-east-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/ubc-eoas/cluster.yaml b/config/clusters/ubc-eoas/cluster.yaml index 7073a0700b..fd1e44c274 100644 --- a/config/clusters/ubc-eoas/cluster.yaml +++ b/config/clusters/ubc-eoas/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: ubc-eoas region: ca-central-1 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml diff --git a/config/clusters/victor/cluster.yaml b/config/clusters/victor/cluster.yaml index 823031a6ab..5ee56d5759 100644 --- a/config/clusters/victor/cluster.yaml +++ b/config/clusters/victor/cluster.yaml @@ -5,6 +5,8 @@ aws: clusterType: eks clusterName: victor region: us-west-2 + billing: + paid_by_us: true support: helm_chart_values_files: - support.values.yaml From 72e7a7c164920fef8ae6f00d8b8c5b68c755388f Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Mar 2024 15:53:27 +0100 Subject: [PATCH 5/5] leap: change paid_by_us to false --- config/clusters/leap/cluster.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/clusters/leap/cluster.yaml b/config/clusters/leap/cluster.yaml index 838e953855..7fb10387cb 100644 --- a/config/clusters/leap/cluster.yaml +++ b/config/clusters/leap/cluster.yaml @@ -6,7 +6,7 @@ gcp: cluster: leap-cluster zone: us-central1 billing: - paid_by_us: true + paid_by_us: false bigquery: project: leap-pangeo dataset: cloud_costs