diff --git a/kubespawner/objects.py b/kubespawner/objects.py index 19a9c58b..a97ccc40 100644 --- a/kubespawner/objects.py +++ b/kubespawner/objects.py @@ -6,7 +6,6 @@ import os import re from urllib.parse import urlparse - from kubernetes.client.models import ( V1Affinity, V1Container, @@ -36,6 +35,8 @@ V1PreferredSchedulingTerm, V1ResourceRequirements, V1Secret, + V1EnvFromSource, + V1SecretEnvSource, V1SecurityContext, V1Service, V1ServicePort, @@ -63,6 +64,7 @@ def make_pod( run_privileged=False, allow_privilege_escalation=True, env=None, + env_from=None, working_dir=None, volumes=None, volume_mounts=None, @@ -266,6 +268,7 @@ def make_pod( pod.spec = V1PodSpec(containers=[]) pod.spec.restart_policy = 'OnFailure' + if image_pull_secrets is not None: # image_pull_secrets as received by the make_pod function should always @@ -288,12 +291,6 @@ def make_pod( } ) - env['JUPYTERHUB_SSL_KEYFILE'] = ssl_secret_mount_path + "ssl.key" - env['JUPYTERHUB_SSL_CERTFILE'] = ssl_secret_mount_path + "ssl.crt" - env['JUPYTERHUB_SSL_CLIENT_CA'] = ( - ssl_secret_mount_path + "notebooks-ca_trust.crt" - ) - if not volume_mounts: volume_mounts = [] volume_mounts.append( @@ -340,24 +337,13 @@ def make_pod( if all([e is None for e in container_security_context.to_dict().values()]): container_security_context = None - # Transform a dict into valid Kubernetes EnvVar Python representations. This - # representation shall always have a "name" field as well as either a - # "value" field or "value_from" field. For examples see the - # test_make_pod_with_env function. - prepared_env = [] - for k, v in (env or {}).items(): - if type(v) == dict: - if not "name" in v: - v["name"] = k - prepared_env.append(get_k8s_model(V1EnvVar, v)) - else: - prepared_env.append(V1EnvVar(name=k, value=v)) notebook_container = V1Container( name='notebook', image=image, working_dir=working_dir, ports=[V1ContainerPort(name='notebook-port', container_port=port)], - env=prepared_env, + env=env, + env_from=[V1EnvFromSource(secret_ref=V1SecretEnvSource(env_from))], args=cmd, image_pull_policy=image_pull_policy, lifecycle=lifecycle_hooks, @@ -671,9 +657,7 @@ def make_owner_reference(name, uid): def make_secret( name, - username, - cert_paths, - hub_ca, + string_data, owner_references, labels=None, annotations=None, @@ -688,10 +672,6 @@ def make_secret( going to be created in. username: The name of the user notebook. - cert_paths: - JupyterHub spawners cert_paths dictionary container certificate path references - hub_ca: - Path to the hub certificate authority labels: Labels to add to the secret. annotations: @@ -706,26 +686,7 @@ def make_secret( secret.metadata.annotations = (annotations or {}).copy() secret.metadata.labels = (labels or {}).copy() secret.metadata.owner_references = owner_references - - secret.data = {} - - with open(cert_paths['keyfile'], 'r') as file: - encoded = base64.b64encode(file.read().encode("utf-8")) - secret.data['ssl.key'] = encoded.decode("utf-8") - - with open(cert_paths['certfile'], 'r') as file: - encoded = base64.b64encode(file.read().encode("utf-8")) - secret.data['ssl.crt'] = encoded.decode("utf-8") - - with open(cert_paths['cafile'], 'r') as file: - encoded = base64.b64encode(file.read().encode("utf-8")) - secret.data["notebooks-ca_trust.crt"] = encoded.decode("utf-8") - - with open(hub_ca, 'r') as file: - encoded = base64.b64encode(file.read().encode("utf-8")) - secret.data["notebooks-ca_trust.crt"] = secret.data[ - "notebooks-ca_trust.crt" - ] + encoded.decode("utf-8") + secret.string_data = string_data return secret diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index d4fceb35..c4234b51 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -7,6 +7,7 @@ import asyncio import json +import base64 import multiprocessing import os import string @@ -27,6 +28,8 @@ from jupyterhub.spawner import Spawner from jupyterhub.traitlets import Command from jupyterhub.utils import exponential_backoff +from kubespawner.utils import get_k8s_model +from kubernetes.client.models import V1EnvVar from kubernetes import client from kubernetes.client.rest import ApiException from slugify import slugify @@ -1624,7 +1627,8 @@ async def get_pod_manifest(self): supplemental_gids=supplemental_gids, run_privileged=self.privileged, allow_privilege_escalation=self.allow_privilege_escalation, - env=self.get_env(), + env=self.get_env_value_from() if self.get_env_value_from() else None, + env_from=self.secret_name, volumes=self._expand_all(self.volumes), volume_mounts=self._expand_all(self.volume_mounts), working_dir=self.working_dir, @@ -1665,12 +1669,10 @@ def get_secret_manifest(self, owner_reference): annotations = self._build_common_annotations( self._expand_all(self.extra_annotations) ) - + return make_secret( name=self.secret_name, - username=self.user.name, - cert_paths=self.cert_paths, - hub_ca=self.internal_trust_bundles['hub-ca'], + string_data=self.get_env(), owner_references=[owner_reference], labels=labels, annotations=annotations, @@ -1768,11 +1770,57 @@ def get_env(self): """ env = super(KubeSpawner, self).get_env() + + # Filter env. variables with only values and pass it to secret + env = {k: v for k, v in (env or {}).items() if type(v) != dict} # deprecate image env['JUPYTER_IMAGE_SPEC'] = self.image env['JUPYTER_IMAGE'] = self.image + + if self.internal_ssl: + """ + cert_paths: + certificate path references + hub_ca: + Path to the hub certificate authority + """ + with open(self.cert_paths['keyfile'], 'r') as file: + env['ssl.key'] = file.read() + + with open(self.cert_paths['certfile'], 'r') as file: + env['ssl.crt'] = file.read() + + with open(self.cert_paths['cafile'], 'r') as file: + env["notebooks-ca_trust.crt"] = file.read() + + with open(self.internal_trust_bundles['hub-ca'], 'r') as file: + env["notebooks-ca_trust.crt"] = env[ + "notebooks-ca_trust.crt" + ] + file.read() + env['JUPYTERHUB_SSL_KEYFILE'] = self.secret_mount_path + "ssl.key" + env['JUPYTERHUB_SSL_CERTFILE'] = self.secret_mount_path + "ssl.crt" + env['JUPYTERHUB_SSL_CLIENT_CA'] = (self.secret_mount_path + "notebooks-ca_trust.crt") return env + + def get_env_value_from(self): + + """Return the environment dict to use for the Spawner. + + See also: jupyterhub.Spawner.get_env + """ + + env = super(KubeSpawner, self).get_env() + env_value_from = [] + # Filter env. variables with only "valueFrom" environment variables. + extra_env = {k: v for k, v in (env or {}).items() if type(v) == dict} + for k, v in (extra_env or {}).items(): + if not "name" in v: + v["name"] = k + env_value_from.append(get_k8s_model(V1EnvVar, v)) + + return env_value_from + def load_state(self, state): """ @@ -2182,34 +2230,38 @@ async def _start(self): timeout=self.k8s_api_request_retry_timeout ) - if self.internal_ssl: - try: - # wait for pod to have uid, - # required for creating owner reference - await exponential_backoff( - lambda: self.pod_has_uid( - self.pod_reflector.pods.get(self.pod_name, None) - ), - "pod/%s does not have a uid!" % (self.pod_name), - ) + try: + # wait for pod to have uid, + # required for creating owner reference + await exponential_backoff( + lambda: self.pod_has_uid( + self.pod_reflector.pods.get(self.pod_name, None) + ), + "pod/%s does not have a uid!" % (self.pod_name), + ) - pod = self.pod_reflector.pods[self.pod_name] - owner_reference = make_owner_reference( - self.pod_name, pod["metadata"]["uid"] - ) + pod = self.pod_reflector.pods[self.pod_name] + owner_reference = make_owner_reference( + self.pod_name, pod["metadata"]["uid"] + ) + + """Create secret object - # internal ssl, create secret object - secret_manifest = self.get_secret_manifest(owner_reference) - await exponential_backoff( - partial( - self._ensure_not_exists, "secret", secret_manifest.metadata.name - ), - f"Failed to delete secret {secret_manifest.metadata.name}", - ) - await exponential_backoff( - partial(self._make_create_resource_request, "secret", secret_manifest), - f"Failed to create secret {secret_manifest.metadata.name}", - ) + Assuming there will be atleast one env. variable that will be passed + """ + secret_manifest = self.get_secret_manifest(owner_reference) + await exponential_backoff( + partial( + self._ensure_not_exists, "secret", secret_manifest.metadata.name + ), + f"Failed to delete secret {secret_manifest.metadata.name}", + ) + await exponential_backoff( + partial(self._make_create_resource_request, "secret", secret_manifest), + f"Failed to create secret {secret_manifest.metadata.name}", + ) + + if self.internal_ssl: service_manifest = self.get_service_manifest(owner_reference) await exponential_backoff( @@ -2224,10 +2276,11 @@ async def _start(self): ), f"Failed to create service {service_manifest.metadata.name}", ) - except Exception: - # cleanup on failure and re-raise - await self.stop(True) - raise + + except Exception: + # cleanup on failure and re-raise + await self.stop(True) + raise # we need a timeout here even though start itself has a timeout # in order for this coroutine to finish at some point. diff --git a/tests/test_objects.py b/tests/test_objects.py index e403e372..d2684131 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -2,7 +2,7 @@ Test functions used to create k8s objects """ from kubernetes.client import ApiClient -from kubespawner.objects import make_ingress, make_pod, make_pvc +from kubespawner.objects import make_ingress, make_pod, make_pvc, make_secret api_client = ApiClient() @@ -26,7 +26,7 @@ def test_make_simplest_pod(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -70,7 +70,7 @@ def test_make_labeled_pod(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -114,7 +114,7 @@ def test_make_annotated_pod(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -162,7 +162,7 @@ def test_make_pod_with_image_pull_secrets_simplified_format(): ], "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -211,7 +211,7 @@ def test_make_pod_with_image_pull_secrets_k8s_native_format(): ], "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -261,7 +261,7 @@ def test_set_container_uid_and_gid(): "runAsUser": 0, "runAsGroup": 0 }, - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -309,7 +309,7 @@ def test_set_container_uid_and_pod_fs_gid(): "securityContext": { "runAsUser": 1000, }, - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -360,7 +360,7 @@ def test_set_pod_supplemental_gids(): "securityContext": { "runAsUser": 1000, }, - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -407,7 +407,7 @@ def test_run_privileged_container(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -454,7 +454,7 @@ def test_allow_privilege_escalation_container(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -507,7 +507,7 @@ def test_make_pod_resources_all(): "nodeSelector": {"disk": "ssd"}, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -537,33 +537,34 @@ def test_make_pod_resources_all(): } -def test_make_pod_with_env(): +def test_make_pod_with_env_from(): """ Test specification of a pod with custom environment variables. """ assert api_client.sanitize_for_serialization(make_pod( name='test', image='jupyter/singleuser:latest', - env={ - 'TEST_KEY_1': 'TEST_VALUE', - 'TEST_KEY_2': { - 'valueFrom': { - 'secretKeyRef': { - 'name': 'my-k8s-secret', - 'key': 'password', + env=[ + { + 'name': 'TEST_KEY_1', + 'valueFrom': { + 'secretKeyRef': { + 'name': 'my-test-secret', + 'key': 'password', + }, }, }, - }, - 'TEST_KEY_NAME_IGNORED': { - 'name': 'TEST_KEY_3', - 'valueFrom': { - 'secretKeyRef': { - 'name': 'my-k8s-secret', - 'key': 'password', + { + 'name': 'TEST_KEY_2', + 'valueFrom': { + 'secretKeyRef': { + 'name': 'my-test-secret', + 'key': 'password', + }, }, }, - }, - }, + ], + env_from='my-k8s-secret', cmd=['jupyterhub-singleuser'], port=8888, image_pull_policy='IfNotPresent' @@ -580,27 +581,30 @@ def test_make_pod_with_env(): "env": [ { 'name': 'TEST_KEY_1', - 'value': 'TEST_VALUE', - }, - { - 'name': 'TEST_KEY_2', 'valueFrom': { 'secretKeyRef': { - 'name': 'my-k8s-secret', + 'name': 'my-test-secret', 'key': 'password', }, }, }, { - 'name': 'TEST_KEY_3', + 'name': 'TEST_KEY_2', 'valueFrom': { 'secretKeyRef': { - 'name': 'my-k8s-secret', + 'name': 'my-test-secret', 'key': 'password', }, }, }, ], + "envFrom": [ + { + 'secretRef': { + 'name': 'my-k8s-secret' + } + } + ], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -652,7 +656,7 @@ def test_make_pod_with_lifecycle(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -717,7 +721,7 @@ def test_make_pod_with_init_containers(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -785,7 +789,7 @@ def test_make_pod_with_extra_container_config(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -851,7 +855,7 @@ def test_make_pod_with_extra_pod_config(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -912,7 +916,7 @@ def test_make_pod_with_extra_containers(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -970,7 +974,7 @@ def test_make_pod_with_extra_resources(): "nodeSelector": {"disk": "ssd"}, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1002,6 +1006,49 @@ def test_make_pod_with_extra_resources(): "apiVersion": "v1" } +def test_make_secret_simple(): + """ + Test specification of the simplest possible secret specification + """ + assert api_client.sanitize_for_serialization(make_secret( + name='my-k8s-secret', + string_data={}, + owner_references=[], + labels={} + )) == { + 'kind': 'Secret', + 'apiVersion': 'v1', + 'metadata': { + 'name': 'my-k8s-secret', + 'annotations': {}, + 'labels': {}, + 'ownerReferences': [] + }, + 'stringData': {} + } + +def test_make_secret_with_data(): + """ + Test specification of the simplest possible secret specification + """ + assert api_client.sanitize_for_serialization(make_secret( + name='my-k8s-secret', + string_data={"TEST1": "VALUE1"}, + owner_references=[], + labels={} + )) == { + 'kind': 'Secret', + 'apiVersion': 'v1', + 'metadata': { + 'name': 'my-k8s-secret', + 'ownerReferences': [], + 'annotations': {}, + 'labels': {} + }, + 'stringData': {"TEST1": "VALUE1"} + } + + def test_make_pvc_simple(): """ Test specification of the simplest possible pvc specification @@ -1125,7 +1172,7 @@ def test_make_pod_with_service_account(): "spec": { "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1171,7 +1218,7 @@ def test_make_pod_with_scheduler_name(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1230,7 +1277,7 @@ def test_make_pod_with_tolerations(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1286,7 +1333,7 @@ def test_make_pod_with_node_affinity_preferred(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1343,7 +1390,7 @@ def test_make_pod_with_node_affinity_required(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1408,7 +1455,7 @@ def test_make_pod_with_pod_affinity_preferred(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1468,7 +1515,7 @@ def test_make_pod_with_pod_affinity_required(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1531,7 +1578,7 @@ def test_make_pod_with_pod_anti_affinity_preferred(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1591,7 +1638,7 @@ def test_make_pod_with_pod_anti_affinity_required(): "automountServiceAccountToken": False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1641,7 +1688,7 @@ def test_make_pod_with_priority_class_name(): 'automountServiceAccountToken': False, "containers": [ { - "env": [], + "envFrom": [{'secretRef': {}}], "name": "notebook", "image": "jupyter/singleuser:latest", "imagePullPolicy": "IfNotPresent", @@ -1755,11 +1802,7 @@ def test_make_pod_with_ssl(): make_pod( name='ssl', image='jupyter/singleuser:latest', - env={ - 'JUPYTERHUB_SSL_KEYFILE': 'TEST_VALUE', - 'JUPYTERHUB_SSL_CERTFILE': 'TEST', - 'JUPYTERHUB_USER': 'TEST', - }, + env_from='ssl', working_dir='/', cmd=['jupyterhub-singleuser'], port=8888, @@ -1777,19 +1820,11 @@ def test_make_pod_with_ssl(): 'automountServiceAccountToken': False, "containers": [ { - "env": [ + "envFrom": [ { - 'name': 'JUPYTERHUB_SSL_KEYFILE', - 'value': '/etc/jupyterhub/ssl/ssl.key', - }, - { - 'name': 'JUPYTERHUB_SSL_CERTFILE', - 'value': '/etc/jupyterhub/ssl/ssl.crt', - }, - {'name': 'JUPYTERHUB_USER', 'value': 'TEST'}, - { - 'name': 'JUPYTERHUB_SSL_CLIENT_CA', - 'value': '/etc/jupyterhub/ssl/notebooks-ca_trust.crt', + 'secretRef': { + 'name': 'ssl' + } }, ], "name": "notebook", diff --git a/tests/test_spawner.py b/tests/test_spawner.py index ffed3673..eabe083e 100644 --- a/tests/test_spawner.py +++ b/tests/test_spawner.py @@ -581,3 +581,22 @@ def test_get_pvc_manifest(): "heritage": "jupyterhub", } assert manifest.spec.selector == {"matchLabels": {"user": "mock-5fname"}} + + +def test_env_value_from(): + c = Config() + + c.KubeSpawner.environment = { + "TEST_KEY_2": {'valueFrom': {'secretKeyRef': {'name': 'my-test-secret', 'key': 'password'}}}, + "TEST_KEY_3": {'valueFrom': {'fieldRef': {'fieldPath': 'metadata.namespace'}}}, + 'TEST_KEY_NAME_IGNORED': {'name': 'TEST_KEY_4', 'valueFrom': { 'secretKeyRef': {'name': 'my-test-secret-2', 'key': 'password'}}} + } + + spawner = KubeSpawner(config=c, _mock=True) + env_value_from = spawner.get_env_value_from() + assert env_value_from[0].name == "TEST_KEY_2" + assert env_value_from[0].value_from == {'secretKeyRef': {'key': 'password', 'name': 'my-test-secret'}} + assert env_value_from[1].name == "TEST_KEY_3" + assert env_value_from[1].value_from == {'fieldRef': {'fieldPath': 'metadata.namespace'}} + assert env_value_from[2].name == "TEST_KEY_4" + assert env_value_from[2].value_from == {'secretKeyRef': {'key': 'password', 'name': 'my-test-secret-2'}}