From c9a11eb8c185cb88a616c4a21364662ab30f4900 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 18 Oct 2024 21:19:44 -0700 Subject: [PATCH 1/6] nasa-veda: Enable dynamic image building This enables dynamic image building for both staging and prod. Fixes https://github.com/2i2c-org/infrastructure/issues/4691 --- config/clusters/nasa-veda/common.values.yaml | 7 +++++++ .../nasa-veda/enc-prod.secret.values.yaml | 17 +++++++++++++--- config/clusters/nasa-veda/prod.values.yaml | 20 +++++++++++++++++++ config/clusters/nasa-veda/staging.values.yaml | 1 - 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/config/clusters/nasa-veda/common.values.yaml b/config/clusters/nasa-veda/common.values.yaml index 593820c137..f90a15dea8 100644 --- a/config/clusters/nasa-veda/common.values.yaml +++ b/config/clusters/nasa-veda/common.values.yaml @@ -123,6 +123,8 @@ basehub: profile_options: image: display_name: Environment + dynamic_image_building: + enabled: true unlisted_choice: enabled: True display_name: "Custom image" @@ -252,6 +254,8 @@ basehub: profile_options: image: display_name: Environment + dynamic_image_building: + enabled: true unlisted_choice: enabled: true display_name: "Custom image" @@ -285,3 +289,6 @@ basehub: scheduling: userScheduler: enabled: true + + binderhub-service: + enabled: true diff --git a/config/clusters/nasa-veda/enc-prod.secret.values.yaml b/config/clusters/nasa-veda/enc-prod.secret.values.yaml index 63c168feb6..b999eb191b 100644 --- a/config/clusters/nasa-veda/enc-prod.secret.values.yaml +++ b/config/clusters/nasa-veda/enc-prod.secret.values.yaml @@ -5,6 +5,17 @@ basehub: GitHubOAuthenticator: client_id: ENC[AES256_GCM,data:H0O+Ha2N/Ax3K6luitZ37k6EQdk=,iv:CP5BGeDMmE6uznVB6utuW78+UeP8Ge5CnDmocZIkSXU=,tag:Q4QG+b8Nq36bbANHIAfpuA==,type:str] client_secret: ENC[AES256_GCM,data:a0Xm3m7cHRCrBdV5q22ohFMYjdolj9dDnQhg9cG5E9W47Toqxwj2Gg==,iv:/1CHsZez3j26A8ODvlYgbjXtaRsKbGraxTzgoF2ZURs=,tag:cN59IZ+9J48QxSJ0IiqsAQ==,type:str] + imagePullSecret: + create: ENC[AES256_GCM,data:VXMGYw==,iv:t29RaK0WDSGUo1KPPIY1S6usrhOzVURi9MS/TjtrZ2c=,tag:gv/lFMGpl8itOtk1FKzVLg==,type:bool] + registry: ENC[AES256_GCM,data:wD9XWkyngg==,iv:p+Wg1KsEn2BOQaVXsCJdnJjJLOhbcuG3I6NAHGNe908=,tag:M6X4ER4BLw0/J6FaKKlEUg==,type:str] + username: ENC[AES256_GCM,data:kj2Vy/zG6JqxzB/dHVZ5ykuH3HgQ7ImGvg==,iv:nFnIPz/JdHNCGPTvzJmQpRpb+5Y3sMMU1h9k0Znf/Oo=,tag:LtW+KyPMbwa5aqgw5s5F2Q==,type:str] + password: ENC[AES256_GCM,data:UFunKFYLFnGaXuTLtHd/FSeuUKATJugvVm437efr7tqBxnTsAi9BY6P9jUoyrI0F/gbETcW8XcZ43ilM5P4vrA==,iv:hS01n8afHI7p7KKduPQEq45U83IT9npF6Rw/f0B7PXQ=,tag:Wf8gSXFJeY/kpcPDAMv4qA==,type:str] + binderhub-service: + buildPodsRegistryCredentials: + password: ENC[AES256_GCM,data:ciFabSeOnYdm0qz82hQbDRhk4zMf3r0lyhWzrmw5pRUs635Kl3jfAW3Edg/C1wmYrk5b4MeGCHUPsZDxI2yZQQ==,iv:Y1XI64eDVQI2MVmlZPS/9hay7JBWZgGxriIXd6PudwY=,tag:i7W7PA0R4s15otnPKdKdyw==,type:str] + config: + DockerRegistry: + password: ENC[AES256_GCM,data:T3G3WXoOvwR6wgQ0L2iC0j/fjWInhVWoce7McZAqdVmkikpSmHd5vub3Jn97VYTo2NjdT+dxcs7VKYcMeIEGyQ==,iv:KsI5iA2b0txa5kGjg8lsSdvMHi5rUbUucApz38aA+R0=,tag:3G2sEmnQSnZg7mxJ8JzoZw==,type:str] sops: kms: [] gcp_kms: @@ -14,8 +25,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2023-01-26T14:57:40Z" - mac: ENC[AES256_GCM,data:57DrEJDR9q4W9O7SJXo/R0XjriaoTmwTSkWtkdAKkXpxIjT37ynN6sSANlk+zi+q8WbdITbk5K6hsHgOnS3VJrKnjTi8mpo0FJ6vAxIRIIr80NSzA1vhWXKn5rvSFWPNL1eh0XJK8HLoqsKBtNLMTis9jgKmAg4lME8uHgh6yxQ=,iv:Gkr554YmGLgT9ScNyxNcHhPq62C3GYph7L+MILyKN2E=,tag:e5k1BNZRscw+IK4eRtfZrA==,type:str] + lastmodified: "2024-10-19T04:19:27Z" + mac: ENC[AES256_GCM,data:/o1UMKj1UkYJZfnnyMde7hvSxGQgConsBKKFBQCQUqW4TyOtv+ajEpuT1gDI8ooWcCetzcdDVvQp+1WwxH862u1T41JwWspqOWmm+QHVPersoasIxqrbX4fhzoWs65Y5vRN2ZeHW1d4AMZvKUhAai/cmLvfcS39ndS/GP5mZ5SU=,iv:6cP1euO6h/jZeFO4SwdUQChzlW9mAVTIVYzN8bHxbpI=,tag:hm+qDxr2ttNZwff6REYSvw==,type:str] pgp: [] unencrypted_suffix: _unencrypted - version: 3.7.3 + version: 3.8.1 diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml index 1b010b9bf9..c6cd2b69a8 100644 --- a/config/clusters/nasa-veda/prod.values.yaml +++ b/config/clusters/nasa-veda/prod.values.yaml @@ -31,3 +31,23 @@ basehub: extraPodConfig: nodeSelector: 2i2c/hub-name: prod + + binderhub-service: + dockerApi: + nodeSelector: + 2i2c/hub-name: prod + config: + KubernetesBuildExecutor: + node_selector: + node.kubernetes.io/instance-type: r5.xlarge + 2i2c/hub-name: prod + BinderHub: + # something like -docker.pkg.dev// for grc.io + # or quay.io/org/repo/cluster-hub/ for quay.io + image_prefix: quay.io/veda-binder/prod- + DockerRegistry: + url: &url https://quay.io + username: &username veda-binder+image_builder + buildPodsRegistryCredentials: + server: *url + username: *username \ No newline at end of file diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml index fd03ef1296..0df5a46f04 100644 --- a/config/clusters/nasa-veda/staging.values.yaml +++ b/config/clusters/nasa-veda/staging.values.yaml @@ -59,7 +59,6 @@ basehub: imagePullSecrets: [{ name: image-pull-secret }] binderhub-service: - enabled: true dockerApi: nodeSelector: 2i2c/hub-name: staging From e00e9a02ad77724ae768d09be36e6a65431ba50d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 05:15:31 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config/clusters/nasa-veda/prod.values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml index c6cd2b69a8..88ae448192 100644 --- a/config/clusters/nasa-veda/prod.values.yaml +++ b/config/clusters/nasa-veda/prod.values.yaml @@ -50,4 +50,4 @@ basehub: username: &username veda-binder+image_builder buildPodsRegistryCredentials: server: *url - username: *username \ No newline at end of file + username: *username From 65359ce90a8d490b2f92f0ebfdc8fbb9e53c6d11 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 27 Nov 2024 21:19:17 -0800 Subject: [PATCH 3/6] Remove explicit pin for smallest instance This should instead put it on any of the hubs that exist --- config/clusters/nasa-veda/prod.values.yaml | 1 - config/clusters/nasa-veda/staging.values.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml index 88ae448192..35e6e69d7f 100644 --- a/config/clusters/nasa-veda/prod.values.yaml +++ b/config/clusters/nasa-veda/prod.values.yaml @@ -39,7 +39,6 @@ basehub: config: KubernetesBuildExecutor: node_selector: - node.kubernetes.io/instance-type: r5.xlarge 2i2c/hub-name: prod BinderHub: # something like -docker.pkg.dev// for grc.io diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml index 0df5a46f04..420e23d71a 100644 --- a/config/clusters/nasa-veda/staging.values.yaml +++ b/config/clusters/nasa-veda/staging.values.yaml @@ -65,7 +65,6 @@ basehub: config: KubernetesBuildExecutor: node_selector: - node.kubernetes.io/instance-type: r5.xlarge 2i2c/hub-name: staging BinderHub: # something like -docker.pkg.dev// for grc.io From 54b7b55e8d2717d4991a2770d517a9972a3653b8 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 2 Dec 2024 21:28:28 -0800 Subject: [PATCH 4/6] Explicitly use r5.xlarge for build pods --- config/clusters/nasa-veda/common.values.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/clusters/nasa-veda/common.values.yaml b/config/clusters/nasa-veda/common.values.yaml index f90a15dea8..b69aaf7f0b 100644 --- a/config/clusters/nasa-veda/common.values.yaml +++ b/config/clusters/nasa-veda/common.values.yaml @@ -292,3 +292,12 @@ basehub: binderhub-service: enabled: true + # Explicitly specify what nodes we want for our builds + # Otherwise we may scale up a larger node than needed + dockerApi: + nodeSelector: + node.kubernetes.io/instance-type: r5.xlarge + config: + KubernetesBuildExecutor: + node_selector: + node.kubernetes.io/instance-type: r5.xlarge \ No newline at end of file From a768063290d8f74209870e5f4d4baec40759ba6b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 2 Dec 2024 21:29:12 -0800 Subject: [PATCH 5/6] Explicitly set repo2docker image Brings in https://github.com/jupyterhub/repo2docker/pull/1380 --- helm-charts/basehub/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-charts/basehub/values.yaml b/helm-charts/basehub/values.yaml index 6289f70050..1afe453309 100644 --- a/helm-charts/basehub/values.yaml +++ b/helm-charts/basehub/values.yaml @@ -42,6 +42,7 @@ binderhub-service: base_url: /services/binder use_registry: true KubernetesBuildExecutor: + build_image: quay.io/jupyterhub/repo2docker:2024.07.0-34.gf77a7d9 node_selector: # Schedule builder pods to run on user nodes only hub.jupyter.org/node-purpose: user From f6ff6e1c9b108851a951fa893e398447983daaeb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 05:29:58 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config/clusters/nasa-veda/common.values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/clusters/nasa-veda/common.values.yaml b/config/clusters/nasa-veda/common.values.yaml index b69aaf7f0b..63a831f22a 100644 --- a/config/clusters/nasa-veda/common.values.yaml +++ b/config/clusters/nasa-veda/common.values.yaml @@ -300,4 +300,4 @@ basehub: config: KubernetesBuildExecutor: node_selector: - node.kubernetes.io/instance-type: r5.xlarge \ No newline at end of file + node.kubernetes.io/instance-type: r5.xlarge