From 98fe35acb2576e42a5df051c1ac2a90c857e9025 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Mon, 19 Dec 2022 10:59:16 -0800 Subject: [PATCH 01/10] Add taintmanager --- .../image-puller/_helpers-daemonset.tpl | 62 ++++++++++++++++++- jupyterhub/templates/image-puller/rbac.yaml | 36 +++++++++++ .../user-placeholder/statefulset.yaml | 2 +- jupyterhub/values.yaml | 11 ++++ 4 files changed, 108 insertions(+), 3 deletions(-) diff --git a/jupyterhub/templates/image-puller/_helpers-daemonset.tpl b/jupyterhub/templates/image-puller/_helpers-daemonset.tpl index 16213b09be..b3018087e5 100644 --- a/jupyterhub/templates/image-puller/_helpers-daemonset.tpl +++ b/jupyterhub/templates/image-puller/_helpers-daemonset.tpl @@ -51,6 +51,8 @@ spec: per node limit all k8s clusters have and have a higher priority than user-placeholder pods that could block an entire node. */}} + serviceAccount: taintmanager + serviceAccountName: taintmanager {{- if .Values.scheduling.podPriority.enabled }} priorityClassName: {{ include "jupyterhub.image-puller-priority.fullname" . }} {{- end }} @@ -58,7 +60,7 @@ spec: nodeSelector: {{- . | toYaml | nindent 8 }} {{- end }} - {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations .Values.prePuller.extraTolerations }} + {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations .Values.prePuller.extraTolerations .Values.prePuller.taintmanager.tolerations }} tolerations: {{- . | toYaml | nindent 8 }} {{- end }} @@ -70,11 +72,39 @@ spec: {{- include "jupyterhub.userNodeAffinityRequired" . | nindent 14 }} {{- end }} terminationGracePeriodSeconds: 0 - automountServiceAccountToken: false + automountServiceAccountToken: true {{- with include "jupyterhub.imagePullSecrets" (dict "root" . "image" .Values.singleuser.image) }} imagePullSecrets: {{ . }} {{- end }} initContainers: + {{- if .Values.prePuller.taintmanager.enabled }} + {{- $taint := first .Values.prePuller.taintmanager.tolerations }} + - name: taintmanager-adding + image: {{ .Values.prePuller.taintmanager.image.name }}:{{ .Values.prePuller.taintmanager.image.tag }} + command: + - /taintmanager + - -add + - {{ $taint.key }}:{{ $taint.effect }} + env: + - name: GODEBUG + value: x509sha1=1 + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.prePuller.resources }} + resources: + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.prePuller.containerSecurityContext }} + securityContext: + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- end }} {{- /* --- Conditionally pull an image all user pods will use in an initContainer --- */}} {{- $blockWithIptables := hasKey .Values.singleuser.cloudMetadata "enabled" | ternary (not .Values.singleuser.cloudMetadata.enabled) .Values.singleuser.cloudMetadata.blockWithIptables }} {{- if $blockWithIptables }} @@ -200,6 +230,34 @@ spec: {{- . | toYaml | nindent 12 }} {{- end }} {{- end }} + {{- if .Values.prePuller.taintmanager.enabled }} + {{- $taint := first .Values.prePuller.taintmanager.tolerations }} + - name: taintmanager-removing + image: {{ .Values.prePuller.taintmanager.image.name }}:{{ .Values.prePuller.taintmanager.image.tag }} + command: + - /taintmanager + - -remove + - {{ $taint.key }}:{{ $taint.effect }} + env: + - name: GODEBUG + value: x509sha1=1 + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.prePuller.resources }} + resources: + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.prePuller.containerSecurityContext }} + securityContext: + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- end }} containers: - name: pause image: {{ .Values.prePuller.pause.image.name }}:{{ .Values.prePuller.pause.image.tag }} diff --git a/jupyterhub/templates/image-puller/rbac.yaml b/jupyterhub/templates/image-puller/rbac.yaml index 996a59a4a3..22f6094a93 100644 --- a/jupyterhub/templates/image-puller/rbac.yaml +++ b/jupyterhub/templates/image-puller/rbac.yaml @@ -42,4 +42,40 @@ roleRef: name: {{ include "jupyterhub.hook-image-awaiter.fullname" . }} apiGroup: rbac.authorization.k8s.io {{- end }} +{{- if .Values.prePuller.taintmanager.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: taintmanager + namespace: "{{ .Release.Namespace }}" + labels: + {{- include "jupyterhub.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: taintmanager + labels: + {{- include "jupyterhub.labels" . | nindent 4 }} +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["nodes"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: taintmanager + labels: + {{- include "jupyterhub.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: taintmanager + namespace: "{{ .Release.Namespace }}" +roleRef: + kind: ClusterRole + name: taintmanager + apiGroup: rbac.authorization.k8s.io +{{- end }} {{- end }} diff --git a/jupyterhub/templates/scheduling/user-placeholder/statefulset.yaml b/jupyterhub/templates/scheduling/user-placeholder/statefulset.yaml index 7f2c785b99..3ae61ab44d 100644 --- a/jupyterhub/templates/scheduling/user-placeholder/statefulset.yaml +++ b/jupyterhub/templates/scheduling/user-placeholder/statefulset.yaml @@ -47,7 +47,7 @@ spec: nodeSelector: {{- . | toYaml | nindent 8 }} {{- end }} - {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations }} + {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations .Values.prePuller.taintmanager.tolerations }} tolerations: {{- . | toYaml | nindent 8 }} {{- end }} diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index 20c630807c..c9fb4ab1fe 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -623,6 +623,17 @@ prePuller: annotations: {} continuous: enabled: true + taintmanager: + enabled: true + image: + name: lthub/taintmanager + tag: master + # taint/tolerations used for preventing user pods being scheduled while user image is being pulled + # only first element in the list is being used + tolerations: + - key: hub.jupyter.org/imagepulling + operator: Exists + effect: NoExecute pullProfileListImages: true extraImages: {} pause: From 08c344fa3743c1da7454174120ea249d6cb0d2d3 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Wed, 4 Jan 2023 11:06:20 -0800 Subject: [PATCH 02/10] Add taintmanager toleration to core pods --- jupyterhub/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index c9fb4ab1fe..f78fecba7a 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -573,6 +573,9 @@ scheduling: operator: Equal value: core effect: NoSchedule + - key: hub.jupyter.org/imagepulling + operator: Exists + effect: NoExecute nodeAffinity: matchNodePurpose: prefer userPods: From 674f5a755b0adf746127c145b09235227e2a3a30 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Fri, 3 Feb 2023 10:34:44 -0800 Subject: [PATCH 03/10] Add taintmanager to schema --- jupyterhub/values.schema.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/jupyterhub/values.schema.yaml b/jupyterhub/values.schema.yaml index 1375a536bb..6426b08b07 100644 --- a/jupyterhub/values.schema.yaml +++ b/jupyterhub/values.schema.yaml @@ -2847,6 +2847,20 @@ properties: properties: enabled: type: boolean + taintmanager: + type: object + additionalProperties: false + required: [enabled] + description: | + Manage the taints during the single user image prepulling. It will add + sepcified taint before image pulling and remove it after it's done. + This will prevent user pods being scheduled on the new node that doesn't + have user image in order to prevent long user wait. + properties: + enabled: + type: boolean + image: *image-spec + tolerations: *tolerations-spec pullProfileListImages: type: boolean description: | From fcb060a5046692ba2cd63724d666df0c1f996866 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Mon, 6 Feb 2023 22:00:19 -0800 Subject: [PATCH 04/10] Remove GODEBUG from taintmanager env var --- jupyterhub/templates/image-puller/_helpers-daemonset.tpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jupyterhub/templates/image-puller/_helpers-daemonset.tpl b/jupyterhub/templates/image-puller/_helpers-daemonset.tpl index b3018087e5..1fee2f982a 100644 --- a/jupyterhub/templates/image-puller/_helpers-daemonset.tpl +++ b/jupyterhub/templates/image-puller/_helpers-daemonset.tpl @@ -86,8 +86,6 @@ spec: - -add - {{ $taint.key }}:{{ $taint.effect }} env: - - name: GODEBUG - value: x509sha1=1 - name: MY_POD_NAME valueFrom: fieldRef: @@ -239,8 +237,6 @@ spec: - -remove - {{ $taint.key }}:{{ $taint.effect }} env: - - name: GODEBUG - value: x509sha1=1 - name: MY_POD_NAME valueFrom: fieldRef: From 3f65fac93a509f5d143e53e22b4da4206a8a204f Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Wed, 8 Feb 2023 00:18:01 -0800 Subject: [PATCH 05/10] Add taint manager The taint manager is a go binary to add/remove taint for a node. --- chartpress.yaml | 4 + images/taint-manager/.gitignore | 2 + images/taint-manager/Dockerfile | 25 ++ images/taint-manager/README.md | 25 ++ images/taint-manager/Tiltfile | 105 ++++++++ images/taint-manager/deployment.yaml | 30 +++ images/taint-manager/go.mod | 46 ++++ images/taint-manager/go.sum | 246 ++++++++++++++++++ images/taint-manager/taintmanager.go | 132 ++++++++++ images/taint-manager/test/clusterrole.yaml | 8 + .../test/clusterrolebinding.yaml | 16 ++ .../test/deployment-busybox.yaml | 36 +++ images/taint-manager/test/sa.yaml | 5 + 13 files changed, 680 insertions(+) create mode 100644 images/taint-manager/.gitignore create mode 100644 images/taint-manager/Dockerfile create mode 100644 images/taint-manager/README.md create mode 100644 images/taint-manager/Tiltfile create mode 100644 images/taint-manager/deployment.yaml create mode 100644 images/taint-manager/go.mod create mode 100644 images/taint-manager/go.sum create mode 100644 images/taint-manager/taintmanager.go create mode 100644 images/taint-manager/test/clusterrole.yaml create mode 100644 images/taint-manager/test/clusterrolebinding.yaml create mode 100644 images/taint-manager/test/deployment-busybox.yaml create mode 100644 images/taint-manager/test/sa.yaml diff --git a/chartpress.yaml b/chartpress.yaml index 6e32a1609e..d18074f0d1 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -58,3 +58,7 @@ charts: # singleuser-sample, a primitive user container to start with. singleuser-sample: valuesPath: singleuser.image + + # taint-manager, an initContainer to add/remove taint for a node + taint-manager: + valuesPath: prePuller.taintmanager.image diff --git a/images/taint-manager/.gitignore b/images/taint-manager/.gitignore new file mode 100644 index 0000000000..0d12116dfb --- /dev/null +++ b/images/taint-manager/.gitignore @@ -0,0 +1,2 @@ +.idea +taintmanager diff --git a/images/taint-manager/Dockerfile b/images/taint-manager/Dockerfile new file mode 100644 index 0000000000..de96bf6b59 --- /dev/null +++ b/images/taint-manager/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 + +## Build +FROM golang:1.18-bullseye AS build + +WORKDIR /app + +COPY go.mod ./ +COPY go.sum ./ +RUN go mod download + +COPY *.go ./ + +RUN go build -o /taintmanager + +## Deploy +FROM gcr.io/distroless/base-debian11 + +WORKDIR / + +COPY --from=build /taintmanager /taintmanager + +USER nonroot:nonroot + +CMD ["/taintmanager"] diff --git a/images/taint-manager/README.md b/images/taint-manager/README.md new file mode 100644 index 0000000000..9ac722d1f8 --- /dev/null +++ b/images/taint-manager/README.md @@ -0,0 +1,25 @@ +In Cluster Taint Manager +========================== + +To add or remove taint of a node from a in-cluster pod. + +Compile +--------- + +``` +GOOS=linux GOARCH=amd64 go build -o taintmanager taintmanager.go +``` + +Development and Debug +----------------------- + +The dev/debug environment is setup by `tilt`. To start, run `tilt up`. + + +Test +------- + +The `test` directory contains YAML files for deploy a pod with required permissions to run taintmanager. +Please change `namespace` field in `clusterrolebinding.yaml` before deploying to a cluster. + +After deploying yaml files, run `kubectl cp` to copy compiled binary to the target pod and run the binary inside pod. diff --git a/images/taint-manager/Tiltfile b/images/taint-manager/Tiltfile new file mode 100644 index 0000000000..3e83509421 --- /dev/null +++ b/images/taint-manager/Tiltfile @@ -0,0 +1,105 @@ +# Welcome to Tilt! +# To get you started as quickly as possible, we have created a +# starter Tiltfile for you. +# +# Uncomment, modify, and delete any commands as needed for your +# project's configuration. + +cluster_name = 'k8scluster' +project_name = 'taintmanager' + +# Allow the cluster to avoid problems while having kubectl configured to talk to a remote cluster. +allow_k8s_contexts(cluster_name) + +# Use custom Dockerfile for Tilt builds, which only takes locally built binary for live reloading. +dockerfile = ''' + FROM golang:1.19-alpine + RUN go install github.com/go-delve/delve/cmd/dlv@latest + COPY %s /usr/local/bin/%s + ''' % (project_name, project_name)# Build Docker image + +# Tilt will automatically associate image builds with the resource(s) +# that reference them (e.g. via Kubernetes or Docker Compose YAML). +# +# More info: https://docs.tilt.dev/api.html#api.docker_build +# +# docker_build('registry.example.com/my-image', +# context='.', +# # (Optional) Use a custom Dockerfile path +# dockerfile='./deploy/app.dockerfile', +# # (Optional) Filter the paths used in the build +# only=['./app'], +# # (Recommended) Updating a running container in-place +# # https://docs.tilt.dev/live_update_reference.html +# live_update=[ +# # Sync files from host to container +# sync('./app', '/src/'), +# # Execute commands inside the container when certain +# # paths change +# run('/src/codegen.sh', trigger=['./app/api']) +# ] +# ) + + +# Run local commands +# Local commands can be helpful for one-time tasks like installing +# project prerequisites. They can also manage long-lived processes +# for non-containerized services or dependencies. +# +# More info: https://docs.tilt.dev/local_resource.html +# +# local_resource('install-helm', +# cmd='which helm > /dev/null || brew install helm', +# # `cmd_bat`, when present, is used instead of `cmd` on Windows. +# cmd_bat=[ +# 'powershell.exe', +# '-Noninteractive', +# '-Command', +# '& {if (!(Get-Command helm -ErrorAction SilentlyContinue)) {scoop install helm}}' +# ] +# ) + +# Building binary locally. +local_resource('%s-binary' % project_name, + 'GOOS=linux GOARCH=amd64 go build -gcflags "all=-N -l" -o taintmanager taintmanager.go', + deps=[ + './taintmanager.go', + ], +) + +# Extensions are open-source, pre-packaged functions that extend Tilt +# +# More info: https://github.com/tilt-dev/tilt-extensions +# +load('ext://git_resource', 'git_checkout') + +# Load the restart_process extension with the docker_build_with_restart func for live reloading. +load('ext://restart_process', 'docker_build_with_restart') + +# Wrap a docker_build to restart the given entrypoint after a Live Update. +docker_build( + 'lthub/' + project_name, + '.', + dockerfile_contents=dockerfile, + entrypoint='/go/bin/dlv --listen=0.0.0.0:50100 --api-version=2 --headless=true --only-same-user=false --accept-multiclient --check-go-version=false exec -- /usr/local/bin/taintmanager -remove jupyterhub:NoSchedule', + live_update=[ + # Copy the binary so it gets restarted. + sync(project_name, '/usr/local/bin/%s' % project_name), + ], +) + +# Apply Kubernetes manifests +# Tilt will build & push any necessary images, re-deploying your +# resources as they change. +# +# More info: https://docs.tilt.dev/api.html#api.k8s_yaml +# +# k8s_yaml(['k8s/deployment.yaml', 'k8s/service.yaml']) +# Create the deployment from YAML file path. +k8s_yaml('deployment.yaml') + +# Configure the resource. +k8s_resource( + project_name, + port_forwards = ["50100:50100"] # Set up the K8s port-forward to be able to connect to it locally. +) diff --git a/images/taint-manager/deployment.yaml b/images/taint-manager/deployment.yaml new file mode 100644 index 0000000000..109d4e68f0 --- /dev/null +++ b/images/taint-manager/deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: taintmanager +spec: + replicas: 1 + selector: + matchLabels: + app: taintmanager + template: + metadata: + labels: + app: taintmanager + spec: + containers: + - image: lthub/taintmanager + imagePullPolicy: IfNotPresent + name: taintmanager + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 8080 + restartPolicy: Always diff --git a/images/taint-manager/go.mod b/images/taint-manager/go.mod new file mode 100644 index 0000000000..aacda2c1f8 --- /dev/null +++ b/images/taint-manager/go.mod @@ -0,0 +1,46 @@ +module taintmanager + +go 1.18 + +require ( + k8s.io/api v0.25.4 + k8s.io/apimachinery v0.25.4 + k8s.io/client-go v0.25.4 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.6.9 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + golang.org/x/net v0.2.0 // indirect + golang.org/x/oauth2 v0.2.0 // indirect + golang.org/x/sys v0.2.0 // indirect + golang.org/x/term v0.2.0 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/time v0.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a // indirect + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/images/taint-manager/go.sum b/images/taint-manager/go.sum new file mode 100644 index 0000000000..2d904547ec --- /dev/null +++ b/images/taint-manager/go.sum @@ -0,0 +1,246 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= +github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU= +golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= +golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= +k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= +k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= +k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= +k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a h1:UR2YSPKAb8j3uL2yK8V+t2ElG4RoBxhJTxa5gg0ZtSo= +k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/images/taint-manager/taintmanager.go b/images/taint-manager/taintmanager.go new file mode 100644 index 0000000000..5e536acb50 --- /dev/null +++ b/images/taint-manager/taintmanager.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "flag" + "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "log" + "os" + "strings" +) + +func main() { + taintAdd := flag.String("add", "", "The taint to add") + taintRemove := flag.String("remove", "", "The taint to remove") + flag.Parse() + tAdd, errAdd := parseTaint(*taintAdd) + tRemove, errRemove := parseTaint(*taintRemove) + + if errAdd != nil && errRemove != nil { + log.Println("Please specify at least one option -add or -remove") + log.Println("Usage: taintmanager [-add TAINT] [-remove TAINT]") + flag.PrintDefaults() + os.Exit(1) + } + + // creates the in-cluster config + config, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + + podName := os.Getenv("MY_POD_NAME") + nodeName := os.Getenv("MY_NODE_NAME") + log.Printf("Pod name: %v and node name %v\n", podName, nodeName) + + // Get node info from API server + node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + log.Printf("Node %v not found\n", nodeName) + } else if statusError, isStatus := err.(*errors.StatusError); isStatus { + log.Printf("Error getting node %v\n", statusError.ErrStatus.Message) + } else if err != nil { + panic(err.Error()) + } else { + log.Printf("Found node %v\n", node.GetName()) + // Get existing taints + for k, v := range node.Spec.Taints { + log.Printf("%v: %v\n", k, v) + } + // Add taint + if errAdd == nil { + node.Spec.Taints = append(node.Spec.Taints, tAdd) + clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) + log.Printf("Taint %v is added to node %v.", tAdd.ToString(), nodeName) + } + // Remove taint + if errRemove == nil { + for i, taint := range node.Spec.Taints { + if taint.Key == tRemove.Key { + node.Spec.Taints = append(node.Spec.Taints[:i], node.Spec.Taints[i+1:]...) + log.Printf("Taint %v is removed from node %v.", tRemove.ToString(), nodeName) + clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) + } + } + } + } +} + +// copied from https://github.com/kubernetes/kubernetes/blob/v1.25.4/pkg/util/taints/taints.go +// parseTaint parses a taint from a string, whose form must be either +// '=:', ':', or ''. +func parseTaint(st string) (v1.Taint, error) { + var taint v1.Taint + + var key string + var value string + var effect v1.TaintEffect + + parts := strings.Split(st, ":") + switch len(parts) { + case 1: + key = parts[0] + case 2: + effect = v1.TaintEffect(parts[1]) + if err := validateTaintEffect(effect); err != nil { + return taint, err + } + + partsKV := strings.Split(parts[0], "=") + if len(partsKV) > 2 { + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + key = partsKV[0] + if len(partsKV) == 2 { + value = partsKV[1] + if errs := validation.IsValidLabelValue(value); len(errs) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + } + default: + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + + if errs := validation.IsQualifiedName(key); len(errs) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + + taint.Key = key + taint.Value = value + taint.Effect = effect + + return taint, nil +} + +func validateTaintEffect(effect v1.TaintEffect) error { + if effect != v1.TaintEffectNoSchedule && effect != v1.TaintEffectPreferNoSchedule && effect != v1.TaintEffectNoExecute { + return fmt.Errorf("invalid taint effect: %v, unsupported taint effect", effect) + } + + return nil +} diff --git a/images/taint-manager/test/clusterrole.yaml b/images/taint-manager/test/clusterrole.yaml new file mode 100644 index 0000000000..2f671740ec --- /dev/null +++ b/images/taint-manager/test/clusterrole.yaml @@ -0,0 +1,8 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: taintmanager +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["nodes"] + verbs: ["get", "update"] diff --git a/images/taint-manager/test/clusterrolebinding.yaml b/images/taint-manager/test/clusterrolebinding.yaml new file mode 100644 index 0000000000..06660a792e --- /dev/null +++ b/images/taint-manager/test/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +# This role binding allows "jane" to read pods in the "default" namespace. +# You need to already have a Role named "pod-reader" in that namespace. +kind: ClusterRoleBinding +metadata: + name: taintmanager +subjects: +# You can specify more than one "subject" +- kind: ServiceAccount + name: taintmanager + namespace: default +roleRef: + # "roleRef" specifies the binding to a Role / ClusterRole + kind: ClusterRole #this must be Role or ClusterRole + name: taintmanager # this must match the name of the Role or ClusterRole you wish to bind to + apiGroup: rbac.authorization.k8s.io diff --git a/images/taint-manager/test/deployment-busybox.yaml b/images/taint-manager/test/deployment-busybox.yaml new file mode 100644 index 0000000000..280e750335 --- /dev/null +++ b/images/taint-manager/test/deployment-busybox.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: taintmanager +spec: + replicas: 1 + selector: + matchLabels: + app: taintmanager + template: + metadata: + labels: + app: taintmanager + spec: + serviceAccountName: taintmanager + containers: + - image: busybox + imagePullPolicy: IfNotPresent + name: taintmanager + command: ["sleep", "100000"] + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 8080 + tolerations: + - key: "hub.jupyter.org/dedicated" + operator: "Exists" + effect: "NoSchedule" + restartPolicy: Always diff --git a/images/taint-manager/test/sa.yaml b/images/taint-manager/test/sa.yaml new file mode 100644 index 0000000000..73784cd76c --- /dev/null +++ b/images/taint-manager/test/sa.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: taintmanager + namespace: default From 50bafb9d5bec478118f05d83b4d9bacdc3db4c4c Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Wed, 8 Feb 2023 00:39:36 -0800 Subject: [PATCH 06/10] Update taint manager image name --- images/taint-manager/Tiltfile | 2 +- images/taint-manager/deployment.yaml | 2 +- jupyterhub/values.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/taint-manager/Tiltfile b/images/taint-manager/Tiltfile index 3e83509421..4f783ce202 100644 --- a/images/taint-manager/Tiltfile +++ b/images/taint-manager/Tiltfile @@ -78,7 +78,7 @@ load('ext://restart_process', 'docker_build_with_restart') # Wrap a docker_build to restart the given entrypoint after a Live Update. docker_build( - 'lthub/' + project_name, + 'jupyterhub/' + project_name, '.', dockerfile_contents=dockerfile, entrypoint='/go/bin/dlv --listen=0.0.0.0:50100 --api-version=2 --headless=true --only-same-user=false --accept-multiclient --check-go-version=false exec -- /usr/local/bin/taintmanager -remove jupyterhub:NoSchedule', diff --git a/images/taint-manager/deployment.yaml b/images/taint-manager/deployment.yaml index 109d4e68f0..af82e8620a 100644 --- a/images/taint-manager/deployment.yaml +++ b/images/taint-manager/deployment.yaml @@ -13,7 +13,7 @@ spec: app: taintmanager spec: containers: - - image: lthub/taintmanager + - image: jupyterhub/k8s-taint-manager imagePullPolicy: IfNotPresent name: taintmanager env: diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index f78fecba7a..2287a2c5cd 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -629,8 +629,8 @@ prePuller: taintmanager: enabled: true image: - name: lthub/taintmanager - tag: master + name: jupyterhub/k8s-taint-manager + tag: "set-by-chartpress" # taint/tolerations used for preventing user pods being scheduled while user image is being pulled # only first element in the list is being used tolerations: From ad399b4ef8999a79282bf0c9e59355aac2beca37 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Wed, 8 Feb 2023 14:09:45 -0800 Subject: [PATCH 07/10] Reformat to make prettier happy --- images/taint-manager/README.md | 13 ++++--------- images/taint-manager/test/clusterrole.yaml | 6 +++--- images/taint-manager/test/clusterrolebinding.yaml | 8 ++++---- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/images/taint-manager/README.md b/images/taint-manager/README.md index 9ac722d1f8..0e8dad27c1 100644 --- a/images/taint-manager/README.md +++ b/images/taint-manager/README.md @@ -1,23 +1,18 @@ -In Cluster Taint Manager -========================== +# In Cluster Taint Manager To add or remove taint of a node from a in-cluster pod. -Compile ---------- +## Compile ``` GOOS=linux GOARCH=amd64 go build -o taintmanager taintmanager.go ``` -Development and Debug ------------------------ +## Development and Debug The dev/debug environment is setup by `tilt`. To start, run `tilt up`. - -Test -------- +## Test The `test` directory contains YAML files for deploy a pod with required permissions to run taintmanager. Please change `namespace` field in `clusterrolebinding.yaml` before deploying to a cluster. diff --git a/images/taint-manager/test/clusterrole.yaml b/images/taint-manager/test/clusterrole.yaml index 2f671740ec..33553ce37c 100644 --- a/images/taint-manager/test/clusterrole.yaml +++ b/images/taint-manager/test/clusterrole.yaml @@ -3,6 +3,6 @@ kind: ClusterRole metadata: name: taintmanager rules: -- apiGroups: [""] # "" indicates the core API group - resources: ["nodes"] - verbs: ["get", "update"] + - apiGroups: [""] # "" indicates the core API group + resources: ["nodes"] + verbs: ["get", "update"] diff --git a/images/taint-manager/test/clusterrolebinding.yaml b/images/taint-manager/test/clusterrolebinding.yaml index 06660a792e..0eb5c77791 100644 --- a/images/taint-manager/test/clusterrolebinding.yaml +++ b/images/taint-manager/test/clusterrolebinding.yaml @@ -5,10 +5,10 @@ kind: ClusterRoleBinding metadata: name: taintmanager subjects: -# You can specify more than one "subject" -- kind: ServiceAccount - name: taintmanager - namespace: default + # You can specify more than one "subject" + - kind: ServiceAccount + name: taintmanager + namespace: default roleRef: # "roleRef" specifies the binding to a Role / ClusterRole kind: ClusterRole #this must be Role or ClusterRole From 7da1b85503bae02ac11f8a5df835f705de158d0e Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Wed, 15 Feb 2023 23:37:40 -0800 Subject: [PATCH 08/10] Add cli parameter for kubeconfig and node name This is to add support for out-cluster development and testing. Also removed Titlefile and deployment manifest as this can be tested outside the cluster --- images/taint-manager/README.md | 15 +- images/taint-manager/Tiltfile | 105 ---------- images/taint-manager/deployment.yaml | 30 --- images/taint-manager/go.mod | 2 + images/taint-manager/go.sum | 3 + images/taint-manager/taintmanager.go | 275 ++++++++++++++++----------- 6 files changed, 177 insertions(+), 253 deletions(-) delete mode 100644 images/taint-manager/Tiltfile delete mode 100644 images/taint-manager/deployment.yaml diff --git a/images/taint-manager/README.md b/images/taint-manager/README.md index 0e8dad27c1..022580a978 100644 --- a/images/taint-manager/README.md +++ b/images/taint-manager/README.md @@ -1,6 +1,8 @@ # In Cluster Taint Manager -To add or remove taint of a node from a in-cluster pod. +To add or remove taint within a pod. The default mode is to run as a pod in a Kubernetes cluster. It will try to +authenticate to K8S API server with in-cluster config (the service account token mounted inside pod). It also uses +downward API to retrieve the node name where the pod is running on in order to change the taint. ## Compile @@ -10,9 +12,16 @@ GOOS=linux GOARCH=amd64 go build -o taintmanager taintmanager.go ## Development and Debug -The dev/debug environment is setup by `tilt`. To start, run `tilt up`. +To test outside a cluster, commandline parameter `-kubeconfig` and `-node` can be specified. If `-kubeconfig` is not +specified, taintmanager will try to load the kubeconfig from default path `HOME/.kube/config` -## Test +Using default kubeconfig: + +``` +taintmanager -node f6.workers.ctlt.ubc.ca -remove hub.jupyter.org/imagepulling:NoExecute +``` + +## Test in Cluster The `test` directory contains YAML files for deploy a pod with required permissions to run taintmanager. Please change `namespace` field in `clusterrolebinding.yaml` before deploying to a cluster. diff --git a/images/taint-manager/Tiltfile b/images/taint-manager/Tiltfile deleted file mode 100644 index 4f783ce202..0000000000 --- a/images/taint-manager/Tiltfile +++ /dev/null @@ -1,105 +0,0 @@ -# Welcome to Tilt! -# To get you started as quickly as possible, we have created a -# starter Tiltfile for you. -# -# Uncomment, modify, and delete any commands as needed for your -# project's configuration. - -cluster_name = 'k8scluster' -project_name = 'taintmanager' - -# Allow the cluster to avoid problems while having kubectl configured to talk to a remote cluster. -allow_k8s_contexts(cluster_name) - -# Use custom Dockerfile for Tilt builds, which only takes locally built binary for live reloading. -dockerfile = ''' - FROM golang:1.19-alpine - RUN go install github.com/go-delve/delve/cmd/dlv@latest - COPY %s /usr/local/bin/%s - ''' % (project_name, project_name)# Build Docker image - -# Tilt will automatically associate image builds with the resource(s) -# that reference them (e.g. via Kubernetes or Docker Compose YAML). -# -# More info: https://docs.tilt.dev/api.html#api.docker_build -# -# docker_build('registry.example.com/my-image', -# context='.', -# # (Optional) Use a custom Dockerfile path -# dockerfile='./deploy/app.dockerfile', -# # (Optional) Filter the paths used in the build -# only=['./app'], -# # (Recommended) Updating a running container in-place -# # https://docs.tilt.dev/live_update_reference.html -# live_update=[ -# # Sync files from host to container -# sync('./app', '/src/'), -# # Execute commands inside the container when certain -# # paths change -# run('/src/codegen.sh', trigger=['./app/api']) -# ] -# ) - - -# Run local commands -# Local commands can be helpful for one-time tasks like installing -# project prerequisites. They can also manage long-lived processes -# for non-containerized services or dependencies. -# -# More info: https://docs.tilt.dev/local_resource.html -# -# local_resource('install-helm', -# cmd='which helm > /dev/null || brew install helm', -# # `cmd_bat`, when present, is used instead of `cmd` on Windows. -# cmd_bat=[ -# 'powershell.exe', -# '-Noninteractive', -# '-Command', -# '& {if (!(Get-Command helm -ErrorAction SilentlyContinue)) {scoop install helm}}' -# ] -# ) - -# Building binary locally. -local_resource('%s-binary' % project_name, - 'GOOS=linux GOARCH=amd64 go build -gcflags "all=-N -l" -o taintmanager taintmanager.go', - deps=[ - './taintmanager.go', - ], -) - -# Extensions are open-source, pre-packaged functions that extend Tilt -# -# More info: https://github.com/tilt-dev/tilt-extensions -# -load('ext://git_resource', 'git_checkout') - -# Load the restart_process extension with the docker_build_with_restart func for live reloading. -load('ext://restart_process', 'docker_build_with_restart') - -# Wrap a docker_build to restart the given entrypoint after a Live Update. -docker_build( - 'jupyterhub/' + project_name, - '.', - dockerfile_contents=dockerfile, - entrypoint='/go/bin/dlv --listen=0.0.0.0:50100 --api-version=2 --headless=true --only-same-user=false --accept-multiclient --check-go-version=false exec -- /usr/local/bin/taintmanager -remove jupyterhub:NoSchedule', - live_update=[ - # Copy the binary so it gets restarted. - sync(project_name, '/usr/local/bin/%s' % project_name), - ], -) - -# Apply Kubernetes manifests -# Tilt will build & push any necessary images, re-deploying your -# resources as they change. -# -# More info: https://docs.tilt.dev/api.html#api.k8s_yaml -# -# k8s_yaml(['k8s/deployment.yaml', 'k8s/service.yaml']) -# Create the deployment from YAML file path. -k8s_yaml('deployment.yaml') - -# Configure the resource. -k8s_resource( - project_name, - port_forwards = ["50100:50100"] # Set up the K8s port-forward to be able to connect to it locally. -) diff --git a/images/taint-manager/deployment.yaml b/images/taint-manager/deployment.yaml deleted file mode 100644 index af82e8620a..0000000000 --- a/images/taint-manager/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: taintmanager -spec: - replicas: 1 - selector: - matchLabels: - app: taintmanager - template: - metadata: - labels: - app: taintmanager - spec: - containers: - - image: jupyterhub/k8s-taint-manager - imagePullPolicy: IfNotPresent - name: taintmanager - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - ports: - - containerPort: 8080 - restartPolicy: Always diff --git a/images/taint-manager/go.mod b/images/taint-manager/go.mod index aacda2c1f8..35bdcff638 100644 --- a/images/taint-manager/go.mod +++ b/images/taint-manager/go.mod @@ -20,12 +20,14 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.2.0 // indirect golang.org/x/oauth2 v0.2.0 // indirect golang.org/x/sys v0.2.0 // indirect diff --git a/images/taint-manager/go.sum b/images/taint-manager/go.sum index 2d904547ec..86b9d35d86 100644 --- a/images/taint-manager/go.sum +++ b/images/taint-manager/go.sum @@ -69,6 +69,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -100,6 +102,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/images/taint-manager/taintmanager.go b/images/taint-manager/taintmanager.go index 5e536acb50..941564bee0 100644 --- a/images/taint-manager/taintmanager.go +++ b/images/taint-manager/taintmanager.go @@ -1,132 +1,177 @@ package main import ( - "context" - "flag" - "fmt" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "log" - "os" - "strings" + "context" + "flag" + "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "log" + "os" + "path/filepath" + "strings" ) +func buildConfig(kubeconfig string) (*rest.Config, error) { + // load kubeconfig from command line + if kubeconfig != "" { + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + log.Printf("Using kubeconfig %v.", kubeconfig) + return cfg, nil + } + + // try the default location HOME/.kube/config + if home := homedir.HomeDir(); home != "" { + kubeconfig = filepath.Join(home, ".kube", "config") + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err == nil { + log.Printf("Using kubeconfig %v.", kubeconfig) + return cfg, nil + } + } + + // try in-cluster auth + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + log.Printf("Using in-cluster config.") + return cfg, nil +} + +func usage() { + log.Println("Usage: taintmanager [-add TAINT] [-remove TAINT] [-node NODE_NAME] [-kubeconfig /PATH/TO/KUBECONFIG]") + flag.PrintDefaults() + os.Exit(1) +} + func main() { - taintAdd := flag.String("add", "", "The taint to add") - taintRemove := flag.String("remove", "", "The taint to remove") - flag.Parse() - tAdd, errAdd := parseTaint(*taintAdd) - tRemove, errRemove := parseTaint(*taintRemove) - - if errAdd != nil && errRemove != nil { - log.Println("Please specify at least one option -add or -remove") - log.Println("Usage: taintmanager [-add TAINT] [-remove TAINT]") - flag.PrintDefaults() - os.Exit(1) - } - - // creates the in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - panic(err.Error()) - } - // creates the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - panic(err.Error()) - } - - podName := os.Getenv("MY_POD_NAME") - nodeName := os.Getenv("MY_NODE_NAME") - log.Printf("Pod name: %v and node name %v\n", podName, nodeName) - - // Get node info from API server - node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - log.Printf("Node %v not found\n", nodeName) - } else if statusError, isStatus := err.(*errors.StatusError); isStatus { - log.Printf("Error getting node %v\n", statusError.ErrStatus.Message) - } else if err != nil { - panic(err.Error()) - } else { - log.Printf("Found node %v\n", node.GetName()) - // Get existing taints - for k, v := range node.Spec.Taints { - log.Printf("%v: %v\n", k, v) - } - // Add taint - if errAdd == nil { - node.Spec.Taints = append(node.Spec.Taints, tAdd) - clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) - log.Printf("Taint %v is added to node %v.", tAdd.ToString(), nodeName) - } - // Remove taint - if errRemove == nil { - for i, taint := range node.Spec.Taints { - if taint.Key == tRemove.Key { - node.Spec.Taints = append(node.Spec.Taints[:i], node.Spec.Taints[i+1:]...) - log.Printf("Taint %v is removed from node %v.", tRemove.ToString(), nodeName) - clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) - } - } - } - } + kubeconfig := flag.String("kubeconfig", "", "(optional) absolute path to the kubeconfig file") + nodeName := flag.String("node", "", "(optional) The name of the node to add/remove taints. Can be passed through environment variable MY_NODE_NAME") + taintAdd := flag.String("add", "", "The taint to add") + taintRemove := flag.String("remove", "", "The taint to remove") + flag.Parse() + + // load kubeconfig + config, err := buildConfig(*kubeconfig) + if err != nil { + log.Printf(err.Error()) + log.Printf("Failed to load kubeconfig") + usage() + } + + // parse taints + tAdd, errAdd := parseTaint(*taintAdd) + tRemove, errRemove := parseTaint(*taintRemove) + if errAdd != nil && errRemove != nil { + log.Println("Please specify at least one option -add or -remove") + usage() + } + + // check node name + if *nodeName == "" { + // try from env var + *nodeName = os.Getenv("MY_NODE_NAME") + } + if *nodeName == "" { + log.Println("Please specify the node name") + usage() + } + log.Printf("Node name %v\n", *nodeName) + + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + node, err := clientset.CoreV1().Nodes().Get(context.TODO(), *nodeName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + log.Printf("Node %v not found in default namespace\n", *nodeName) + } else if statusError, isStatus := err.(*errors.StatusError); isStatus { + log.Printf("Error getting node %v\n", statusError.ErrStatus.Message) + } else if err != nil { + panic(err.Error()) + } else { + log.Printf("Found node %v\n", node.GetName()) + for k, v := range node.Spec.Taints { + log.Printf("%v: %v\n", k, v) + } + if errAdd == nil { + node.Spec.Taints = append(node.Spec.Taints, tAdd) + clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) + log.Printf("Taint %v is added to node %v.", tAdd.ToString(), *nodeName) + } + if errRemove == nil { + for i, taint := range node.Spec.Taints { + if taint.Key == tRemove.Key { + node.Spec.Taints = append(node.Spec.Taints[:i], node.Spec.Taints[i+1:]...) + log.Printf("Taint %v is removed from node %v.", tRemove.ToString(), *nodeName) + clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) + } + } + } + } } // copied from https://github.com/kubernetes/kubernetes/blob/v1.25.4/pkg/util/taints/taints.go // parseTaint parses a taint from a string, whose form must be either // '=:', ':', or ''. func parseTaint(st string) (v1.Taint, error) { - var taint v1.Taint - - var key string - var value string - var effect v1.TaintEffect - - parts := strings.Split(st, ":") - switch len(parts) { - case 1: - key = parts[0] - case 2: - effect = v1.TaintEffect(parts[1]) - if err := validateTaintEffect(effect); err != nil { - return taint, err - } - - partsKV := strings.Split(parts[0], "=") - if len(partsKV) > 2 { - return taint, fmt.Errorf("invalid taint spec: %v", st) - } - key = partsKV[0] - if len(partsKV) == 2 { - value = partsKV[1] - if errs := validation.IsValidLabelValue(value); len(errs) > 0 { - return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) - } - } - default: - return taint, fmt.Errorf("invalid taint spec: %v", st) - } - - if errs := validation.IsQualifiedName(key); len(errs) > 0 { - return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) - } - - taint.Key = key - taint.Value = value - taint.Effect = effect - - return taint, nil + var taint v1.Taint + + var key string + var value string + var effect v1.TaintEffect + + parts := strings.Split(st, ":") + switch len(parts) { + case 1: + key = parts[0] + case 2: + effect = v1.TaintEffect(parts[1]) + if err := validateTaintEffect(effect); err != nil { + return taint, err + } + + partsKV := strings.Split(parts[0], "=") + if len(partsKV) > 2 { + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + key = partsKV[0] + if len(partsKV) == 2 { + value = partsKV[1] + if errs := validation.IsValidLabelValue(value); len(errs) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + } + default: + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + + if errs := validation.IsQualifiedName(key); len(errs) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + + taint.Key = key + taint.Value = value + taint.Effect = effect + + return taint, nil } func validateTaintEffect(effect v1.TaintEffect) error { - if effect != v1.TaintEffectNoSchedule && effect != v1.TaintEffectPreferNoSchedule && effect != v1.TaintEffectNoExecute { - return fmt.Errorf("invalid taint effect: %v, unsupported taint effect", effect) - } + if effect != v1.TaintEffectNoSchedule && effect != v1.TaintEffectPreferNoSchedule && effect != v1.TaintEffectNoExecute { + return fmt.Errorf("invalid taint effect: %v, unsupported taint effect", effect) + } - return nil + return nil } From c32c90d71d7ca3f3e3e0ddad3daf83defd4b4fc5 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Tue, 18 Apr 2023 22:33:14 -0700 Subject: [PATCH 09/10] Fix a typo --- jupyterhub/values.schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/values.schema.yaml b/jupyterhub/values.schema.yaml index 6426b08b07..b44ea00321 100644 --- a/jupyterhub/values.schema.yaml +++ b/jupyterhub/values.schema.yaml @@ -2853,7 +2853,7 @@ properties: required: [enabled] description: | Manage the taints during the single user image prepulling. It will add - sepcified taint before image pulling and remove it after it's done. + specified taint before image pulling and remove it after it's done. This will prevent user pods being scheduled on the new node that doesn't have user image in order to prevent long user wait. properties: From c98495a2692ffc553cad1ad12e696b940d394779 Mon Sep 17 00:00:00 2001 From: Pan Luo Date: Wed, 3 Apr 2024 10:44:51 -0700 Subject: [PATCH 10/10] Change taintmanager image location to make chartpress happy --- jupyterhub/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index 2287a2c5cd..8cc710ed9a 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -629,7 +629,7 @@ prePuller: taintmanager: enabled: true image: - name: jupyterhub/k8s-taint-manager + name: quay.io/jupyterhub/k8s-taint-manager tag: "set-by-chartpress" # taint/tolerations used for preventing user pods being scheduled while user image is being pulled # only first element in the list is being used