From 77efd207cc3b0b1862428ec90845eb17af1134ef Mon Sep 17 00:00:00 2001 From: Brian Freitag Date: Mon, 18 Mar 2024 10:19:09 -0500 Subject: [PATCH 1/7] add s3 permissions for ESDIS hub --- terraform/aws/projects/nasa-esdis.tfvars | 60 +++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/terraform/aws/projects/nasa-esdis.tfvars b/terraform/aws/projects/nasa-esdis.tfvars index d97271f44..d2a4cfe73 100644 --- a/terraform/aws/projects/nasa-esdis.tfvars +++ b/terraform/aws/projects/nasa-esdis.tfvars @@ -21,6 +21,62 @@ hub_cloud_permissions = { }, "prod" : { bucket_admin_access : ["scratch"], - extra_iam_policy : "" + extra_iam_policy : <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucketVersions", + "s3:CreateBucket", + "s3:ListBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::veda-data-store-staging", + "arn:aws:s3:::veda-data-store-staging/*", + "arn:aws:s3:::veda-nex-gddp-cmip6-public", + "arn:aws:s3:::veda-nex-gddp-cmip6-public/*", + "arn:aws:s3:::cmip6-staging", + "arn:aws:s3:::cmip6-staging/*", + "arn:aws:s3:::lp-prod-protected", + "arn:aws:s3:::lp-prod-protected/*", + "arn:aws:s3:::gesdisc-cumulus-prod-protected", + "arn:aws:s3:::gesdisc-cumulus-prod-protected/*", + "arn:aws:s3:::nsidc-cumulus-prod-protected", + "arn:aws:s3:::nsidc-cumulus-prod-protected/*", + "arn:aws:s3:::ornl-cumulus-prod-protected", + "arn:aws:s3:::ornl-cumulus-prod-protected/*", + "arn:aws:s3:::pangeo-forge-veda-output", + "arn:aws:s3:::pangeo-forge-veda-output/*", + "arn:aws:s3:::podaac-ops-cumulus-public", + "arn:aws:s3:::podaac-ops-cumulus-public/*", + "arn:aws:s3:::podaac-ops-cumulus-protected", + "arn:aws:s3:::podaac-ops-cumulus-protected/*", + "arn:aws:s3:::maap-ops-workspace", + "arn:aws:s3:::maap-ops-workspace/*", + "arn:aws:s3:::nasa-maap-data-store", + "arn:aws:s3:::nasa-maap-data-store/*", + "arn:aws::s3:::grss-veda-data-store", + "arn:aws::s3:::grss-veda-data-store/*", + "arn:aws:s3:::sentinel-cogs", + "arn:aws:s3:::sentinel-cogs/*", + ] + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + EOT }, -} \ No newline at end of file +} From cdd1cfbd9d7980849039a3fb055d7e4ee41803f2 Mon Sep 17 00:00:00 2001 From: Brian Freitag Date: Mon, 18 Mar 2024 10:41:46 -0500 Subject: [PATCH 2/7] Update and rebrand ESDIS hub to support IEEE-GRSS workshop --- config/clusters/nasa-esdis/common.values.yaml | 94 ++++++++++--------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/config/clusters/nasa-esdis/common.values.yaml b/config/clusters/nasa-esdis/common.values.yaml index 26bc51df9..fd659b86f 100644 --- a/config/clusters/nasa-esdis/common.values.yaml +++ b/config/clusters/nasa-esdis/common.values.yaml @@ -20,9 +20,9 @@ jupyterhub: homepage: templateVars: org: - name: ESDIS - logo_url: "https://github.com/2i2c-org/infrastructure/assets/61120/3380676a-1f2e-400d-8471-79496510c1e7" - url: https://www.earthdata.nasa.gov/esdis + name: IEEE-GRSS + logo_url: "https://www.grss-ieee.org/wp-content/uploads/2020/12/grss.png" + url: https://www.grss-ieee.org/ designed_by: name: 2i2c url: https://2i2c.org @@ -38,20 +38,58 @@ jupyterhub: authenticator_class: github GitHubOAuthenticator: allowed_organizations: - - nasa-esdis:cloud-users + - nasa-veda-workshops:ieee-grss-webinar-mar-2024 scope: - read:org Authenticator: admin_users: - bilts # Patrick Quinn - freitagb # Brian Freitag + - slesaad # Slesa Adhikari singleuser: - profileList: - - display_name: Python - description: Python datascience environment - default: true - kubespawner_override: - image: openscapes/python:6ee57a9 + defaultUrl: /lab + profileList: + - display_name: "Modified Pangeo Notebook" + slug: modified-pangeo + description: Pangeo based notebook with a Python environment + default: true + kubespawner_override: + image: public.ecr.aws/nasa-veda/nasa-veda-singleuser:2024-03-07 + init_containers: + # Need to explicitly fix ownership here, as otherwise these directories will be owned + # by root on most NFS filesystems - neither EFS nor Google Filestore support anonuid + - name: volume-mount-ownership-fix + image: busybox:1.36.1 + command: + - sh + - -c + - id && chown 1000:1000 /home/jovyan /home/jovyan/shared && ls -lhd /home/jovyan + securityContext: + runAsUser: 0 + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" + # mounted without readonly attribute here, + # so we can chown it appropriately + - name: home + mountPath: /home/jovyan/shared + subPath: _shared + # this container uses nbgitpuller to mount https://github.com/NASA-IMPACT/veda-docs/ for user pods + # image source: https://github.com/NASA-IMPACT/veda-jh-environments/tree/main/docker-images/base/nasa-veda-singleuser-init + - name: nasa-veda-singleuser-init + image: public.ecr.aws/nasa-veda/nasa-veda-singleuser-init:38e8998f9be64b0a59ac6c4d6d152d3403121dfc4be6d49bdf52ddc92827af8a + command: + - "python3" + - "/opt/k8s-init-container-nb-docs.py" + - "/home/jovyan" + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 profile_options: &profile_options requests: display_name: Resource Allocation @@ -84,42 +122,6 @@ jupyterhub: cpu_limit: 3.75 node_selector: node.kubernetes.io/instance-type: r5.xlarge - mem_14_8: - display_name: 14.8 GB RAM, upto 3.7 CPUs - kubespawner_override: - mem_guarantee: 15941615616 - mem_limit: 15941615616 - cpu_guarantee: 1.875 - cpu_limit: 3.75 - node_selector: - node.kubernetes.io/instance-type: r5.xlarge - mem_29_7: - display_name: 29.7 GB RAM, upto 3.7 CPUs - kubespawner_override: - mem_guarantee: 31883231232 - mem_limit: 31883231232 - cpu_guarantee: 3.75 - cpu_limit: 3.75 - node_selector: - node.kubernetes.io/instance-type: r5.xlarge - mem_60_6: - display_name: 60.6 GB RAM, upto 15.7 CPUs - kubespawner_override: - mem_guarantee: 65094813696 - mem_limit: 65094813696 - cpu_guarantee: 7.86 - cpu_limit: 15.72 - node_selector: - node.kubernetes.io/instance-type: r5.4xlarge - mem_121_2: - display_name: 121.2 GB RAM, upto 15.7 CPUs - kubespawner_override: - mem_guarantee: 130189627392 - mem_limit: 130189627392 - cpu_guarantee: 15.72 - cpu_limit: 15.72 - node_selector: - node.kubernetes.io/instance-type: r5.4xlarge - display_name: R description: R (with RStudio) + Python environment kubespawner_override: From eee5222f9921544a07ce9492625d3b7f10ceca63 Mon Sep 17 00:00:00 2001 From: Brian Freitag Date: Mon, 18 Mar 2024 10:42:35 -0500 Subject: [PATCH 3/7] update ESDIS hub url to GRSS hub url --- config/clusters/nasa-esdis/cluster.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/clusters/nasa-esdis/cluster.yaml b/config/clusters/nasa-esdis/cluster.yaml index 98d9b3ea0..b9b6e7f85 100644 --- a/config/clusters/nasa-esdis/cluster.yaml +++ b/config/clusters/nasa-esdis/cluster.yaml @@ -20,8 +20,8 @@ hubs: - staging.values.yaml - enc-staging.secret.values.yaml - name: prod - display_name: "ESDIS" - domain: esdis.2i2c.cloud + display_name: "IEEE-GRSS" + domain: ieee-grss.2i2c.cloud helm_chart: basehub helm_chart_values_files: - common.values.yaml From 2c0dafc0c7690818541972a871ea646fb6a9dfef Mon Sep 17 00:00:00 2001 From: Sarah Gibson <44771837+sgibson91@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:12:06 +0000 Subject: [PATCH 4/7] Revert URL and display_name change back to ESDIS - display_name only affects what engineers and the deployer runs on the command line, not user facing - agreed in https://github.com/2i2c-org/infrastructure/pull/3809#issuecomment-2007888019 to skip the URL change since this triggers other non-config related changes, such as updating OAuth apps --- config/clusters/nasa-esdis/cluster.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/clusters/nasa-esdis/cluster.yaml b/config/clusters/nasa-esdis/cluster.yaml index b9b6e7f85..98d9b3ea0 100644 --- a/config/clusters/nasa-esdis/cluster.yaml +++ b/config/clusters/nasa-esdis/cluster.yaml @@ -20,8 +20,8 @@ hubs: - staging.values.yaml - enc-staging.secret.values.yaml - name: prod - display_name: "IEEE-GRSS" - domain: ieee-grss.2i2c.cloud + display_name: "ESDIS" + domain: esdis.2i2c.cloud helm_chart: basehub helm_chart_values_files: - common.values.yaml From d18e38199e96aede7716911b9f64e3d547a179d7 Mon Sep 17 00:00:00 2001 From: Sarah Gibson <44771837+sgibson91@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:13:39 +0000 Subject: [PATCH 5/7] Remove trailing comma --- terraform/aws/projects/nasa-esdis.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/aws/projects/nasa-esdis.tfvars b/terraform/aws/projects/nasa-esdis.tfvars index d2a4cfe73..eeaccf7c5 100644 --- a/terraform/aws/projects/nasa-esdis.tfvars +++ b/terraform/aws/projects/nasa-esdis.tfvars @@ -67,7 +67,7 @@ hub_cloud_permissions = { "arn:aws::s3:::grss-veda-data-store", "arn:aws::s3:::grss-veda-data-store/*", "arn:aws:s3:::sentinel-cogs", - "arn:aws:s3:::sentinel-cogs/*", + "arn:aws:s3:::sentinel-cogs/*" ] }, { From f4d9c29db6f8b1357cb3f719d7d01232aa1b20a2 Mon Sep 17 00:00:00 2001 From: Sarah Gibson <44771837+sgibson91@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:24:13 +0000 Subject: [PATCH 6/7] correct indentation --- config/clusters/nasa-esdis/common.values.yaml | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/config/clusters/nasa-esdis/common.values.yaml b/config/clusters/nasa-esdis/common.values.yaml index fd659b86f..118e58b66 100644 --- a/config/clusters/nasa-esdis/common.values.yaml +++ b/config/clusters/nasa-esdis/common.values.yaml @@ -47,49 +47,49 @@ jupyterhub: - freitagb # Brian Freitag - slesaad # Slesa Adhikari singleuser: - defaultUrl: /lab - profileList: - - display_name: "Modified Pangeo Notebook" - slug: modified-pangeo - description: Pangeo based notebook with a Python environment - default: true - kubespawner_override: - image: public.ecr.aws/nasa-veda/nasa-veda-singleuser:2024-03-07 - init_containers: - # Need to explicitly fix ownership here, as otherwise these directories will be owned - # by root on most NFS filesystems - neither EFS nor Google Filestore support anonuid - - name: volume-mount-ownership-fix - image: busybox:1.36.1 - command: - - sh - - -c - - id && chown 1000:1000 /home/jovyan /home/jovyan/shared && ls -lhd /home/jovyan - securityContext: - runAsUser: 0 - volumeMounts: - - name: home - mountPath: /home/jovyan - subPath: "{username}" - # mounted without readonly attribute here, - # so we can chown it appropriately - - name: home - mountPath: /home/jovyan/shared - subPath: _shared - # this container uses nbgitpuller to mount https://github.com/NASA-IMPACT/veda-docs/ for user pods - # image source: https://github.com/NASA-IMPACT/veda-jh-environments/tree/main/docker-images/base/nasa-veda-singleuser-init - - name: nasa-veda-singleuser-init - image: public.ecr.aws/nasa-veda/nasa-veda-singleuser-init:38e8998f9be64b0a59ac6c4d6d152d3403121dfc4be6d49bdf52ddc92827af8a - command: - - "python3" - - "/opt/k8s-init-container-nb-docs.py" - - "/home/jovyan" - volumeMounts: - - name: home - mountPath: /home/jovyan - subPath: "{username}" - securityContext: - runAsUser: 1000 - runAsGroup: 1000 + defaultUrl: /lab + profileList: + - display_name: "Modified Pangeo Notebook" + slug: modified-pangeo + description: Pangeo based notebook with a Python environment + default: true + kubespawner_override: + image: public.ecr.aws/nasa-veda/nasa-veda-singleuser:2024-03-07 + init_containers: + # Need to explicitly fix ownership here, as otherwise these directories will be owned + # by root on most NFS filesystems - neither EFS nor Google Filestore support anonuid + - name: volume-mount-ownership-fix + image: busybox:1.36.1 + command: + - sh + - -c + - id && chown 1000:1000 /home/jovyan /home/jovyan/shared && ls -lhd /home/jovyan + securityContext: + runAsUser: 0 + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" + # mounted without readonly attribute here, + # so we can chown it appropriately + - name: home + mountPath: /home/jovyan/shared + subPath: _shared + # this container uses nbgitpuller to mount https://github.com/NASA-IMPACT/veda-docs/ for user pods + # image source: https://github.com/NASA-IMPACT/veda-jh-environments/tree/main/docker-images/base/nasa-veda-singleuser-init + - name: nasa-veda-singleuser-init + image: public.ecr.aws/nasa-veda/nasa-veda-singleuser-init:38e8998f9be64b0a59ac6c4d6d152d3403121dfc4be6d49bdf52ddc92827af8a + command: + - "python3" + - "/opt/k8s-init-container-nb-docs.py" + - "/home/jovyan" + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 profile_options: &profile_options requests: display_name: Resource Allocation From fa9819ad440ad4759457cc4af840558717970df0 Mon Sep 17 00:00:00 2001 From: Sarah Gibson <44771837+sgibson91@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:37:01 +0000 Subject: [PATCH 7/7] Remove extra colons --- terraform/aws/projects/nasa-esdis.tfvars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/aws/projects/nasa-esdis.tfvars b/terraform/aws/projects/nasa-esdis.tfvars index eeaccf7c5..9beb0b55c 100644 --- a/terraform/aws/projects/nasa-esdis.tfvars +++ b/terraform/aws/projects/nasa-esdis.tfvars @@ -64,8 +64,8 @@ hub_cloud_permissions = { "arn:aws:s3:::maap-ops-workspace/*", "arn:aws:s3:::nasa-maap-data-store", "arn:aws:s3:::nasa-maap-data-store/*", - "arn:aws::s3:::grss-veda-data-store", - "arn:aws::s3:::grss-veda-data-store/*", + "arn:aws:s3:::grss-veda-data-store", + "arn:aws:s3:::grss-veda-data-store/*", "arn:aws:s3:::sentinel-cogs", "arn:aws:s3:::sentinel-cogs/*" ]