Merge pull request #91 from utilitywarehouse/kubelet-binary

Run kubelet binary directly instead of containerized
utilitywarehouse · Jul 17, 2019 · 56ee749 · 56ee749
2 parents 3d42b4a + f55b7e5
commit 56ee749
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 136 deletions.
diff --git a/common.tf b/common.tf
@@ -65,3 +65,13 @@ data "ignition_file" "format-and-mount" {
     content = file("${path.module}/resources/format-and-mount")
   }
 }
+
+data "ignition_file" "kubelet" {
+  mode       = 493
+  filesystem = "root"
+  path       = "/opt/bin/kubelet"
+
+  source {
+    source = "https://storage.googleapis.com/kubernetes-release/release/${var.hyperkube_image_tag}/bin/linux/amd64/kubelet"
+  }
+}
diff --git a/master.tf b/master.tf
@@ -178,9 +178,8 @@ data "template_file" "master-kubelet" {
   template = file("${path.module}/resources/master-kubelet.service")
 
   vars = {
-    kubelet_image_url = var.hyperkube_image_url
-    kubelet_image_tag = var.hyperkube_image_tag
-    cloud_provider    = var.cloud_provider
+    kubelet_binary_path = "/opt/bin/kubelet"
+    cloud_provider      = var.cloud_provider
   }
 }
 
@@ -390,39 +389,40 @@ locals {
 
 data "ignition_config" "master" {
   files = concat(
-      [
-        data.ignition_file.audit-policy.id,
-        data.ignition_file.cfssl.id,
-        data.ignition_file.cfssljson.id,
-        data.ignition_file.cfssl-client-config.id,
-        data.ignition_file.master-cfssl-new-node-cert.id,
-        data.ignition_file.master-cfssl-new-apiserver-cert.id,
-        data.ignition_file.master-cfssl-new-apiserver-kubelet-client-cert.id,
-        data.ignition_file.master-cfssl-new-scheduler-cert.id,
-        data.ignition_file.master-cfssl-new-controller-manager-cert.id,
-        data.ignition_file.master-cfssl-keys-and-certs-get.id,
-        data.ignition_file.master-prom-machine-role.id,
-        data.ignition_file.scheduler-kubeconfig.id,
-        data.ignition_file.controller-manager-kubeconfig.id,
-        data.ignition_file.kubelet-kubeconfig.id,
-        data.ignition_file.kube-apiserver.id,
-        data.ignition_file.kube-scheduler.id,
-        data.ignition_file.kube-scheduler-config.id,
-        data.ignition_file.kube-controller-manager.id,
-        data.ignition_file.master-kubelet-conf.id,
-      ],
-      var.master_additional_files,
-      [local.kube_controller_additional_config]
-    )
+    [
+      data.ignition_file.audit-policy.id,
+      data.ignition_file.cfssl.id,
+      data.ignition_file.cfssljson.id,
+      data.ignition_file.cfssl-client-config.id,
+      data.ignition_file.master-cfssl-new-node-cert.id,
+      data.ignition_file.master-cfssl-new-apiserver-cert.id,
+      data.ignition_file.master-cfssl-new-apiserver-kubelet-client-cert.id,
+      data.ignition_file.master-cfssl-new-scheduler-cert.id,
+      data.ignition_file.master-cfssl-new-controller-manager-cert.id,
+      data.ignition_file.master-cfssl-keys-and-certs-get.id,
+      data.ignition_file.master-prom-machine-role.id,
+      data.ignition_file.scheduler-kubeconfig.id,
+      data.ignition_file.controller-manager-kubeconfig.id,
+      data.ignition_file.kubelet-kubeconfig.id,
+      data.ignition_file.kube-apiserver.id,
+      data.ignition_file.kube-scheduler.id,
+      data.ignition_file.kube-scheduler-config.id,
+      data.ignition_file.kube-controller-manager.id,
+      data.ignition_file.kubelet.id,
+      data.ignition_file.master-kubelet-conf.id,
+    ],
+    var.master_additional_files,
+    [local.kube_controller_additional_config]
+  )
 
   systemd = concat(
-      [
-        data.ignition_systemd_unit.update-engine.id,
-        data.ignition_systemd_unit.locksmithd_master.id,
-        data.ignition_systemd_unit.docker-opts-dropin.id,
-        data.ignition_systemd_unit.master-kubelet.id,
-      ],
-      module.kubelet-restarter.systemd_units,
-      var.master_additional_systemd_units
-    )
+    [
+      data.ignition_systemd_unit.update-engine.id,
+      data.ignition_systemd_unit.locksmithd_master.id,
+      data.ignition_systemd_unit.docker-opts-dropin.id,
+      data.ignition_systemd_unit.master-kubelet.id,
+    ],
+    module.kubelet-restarter.systemd_units,
+    var.master_additional_systemd_units
+  )
 }
diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service
@@ -1,4 +1,3 @@
-# https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service
 [Unit]
 Description=Kubernetes Kubelet
 Requires=docker.service
@@ -25,40 +24,20 @@ ExecStartPre=/opt/bin/cfssl-new-scheduler-cert
 ExecStartPre=/opt/bin/cfssl-new-controller-manager-cert
 ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-controller-manager' | awk '{ print $1; }')"
 ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-apiserver' | awk '{ print $1; }')"
-ExecStart=/usr/bin/docker \
-  run \
-    --rm \
-    --net host \
-    --pid host \
-    --privileged \
-    --volume /dev:/dev:rw \
-    --volume /sys:/sys:ro \
-    --volume /var/run:/var/run:rw \
-    --volume /var/lib/cni/:/var/lib/cni:rw \
-    --volume /var/lib/docker/:/var/lib/docker:rw \
-    --volume /var/lib/kubelet/:/var/lib/kubelet:shared \
-    --volume /var/log:/var/log:shared \
-    --volume /etc/kubernetes:/etc/kubernetes:ro \
-    --volume /etc/cni/net.d:/etc/cni/net.d:rw \
-    --volume /etc/resolv.conf:/etc/resolv.conf:ro \
-    --volume /opt/cni/bin:/opt/cni/bin:rw \
-    --volume /var/run/calico:/var/run/calico:rw \
-    --volume /var/lib/calico:/var/lib/calico:rw \
-    --entrypoint /usr/local/bin/kubelet \
-  "${kubelet_image_url}:${kubelet_image_tag}" \
-    --allow-privileged \
-    --config=/etc/kubernetes/config/master-kubelet-conf.yaml \
-    --kubeconfig=/var/lib/kubelet/kubeconfig \
-    --node-labels=role=master,node-role.kubernetes.io/master="" \
-    --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
-    --container-runtime=docker \
-    --network-plugin=cni \
-    --cni-bin-dir=/opt/cni/bin \
-    --cni-conf-dir=/etc/cni/net.d \
-    ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
-    --lock-file=/var/run/lock/kubelet.lock \
-    --exit-on-lock-contention \
-    --v=0
+ExecStart=${kubelet_binary_path} \
+  --allow-privileged \
+  --config=/etc/kubernetes/config/master-kubelet-conf.yaml \
+  --kubeconfig=/var/lib/kubelet/kubeconfig \
+  --node-labels=role=master,node-role.kubernetes.io/master="" \
+  --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
+  --container-runtime=docker \
+  --network-plugin=cni \
+  --cni-bin-dir=/opt/cni/bin \
+  --cni-conf-dir=/etc/cni/net.d \
+  ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
+  --lock-file=/var/run/lock/kubelet.lock \
+  --exit-on-lock-contention \
+  --v=0
 Restart=always
 RestartSec=10
 [Install]

diff --git a/resources/worker-kubelet.service b/resources/worker-kubelet.service
@@ -1,4 +1,3 @@
-# https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service
 [Unit]
 Description=Kubernetes Kubelet
 Requires=docker.service
@@ -18,41 +17,19 @@ ExecStartPre=/sbin/sysctl -w net.ipv4.tcp_retries2=8
 #  https://github.com/kubernetes/kubernetes/issues/69015
 ExecStartPre=/sbin/sysctl -w fs.inotify.max_user_watches=524288
 ExecStartPre=/opt/bin/cfssl-new-cert
-ExecStart=/usr/bin/docker \
-  run \
-    --rm \
-    --net host \
-    --pid host \
-    --privileged \
-    --volume /dev:/dev:rw \
-    --volume /sys:/sys:ro \
-    --volume /var/run:/var/run:rw \
-    --volume /var/lib/cni/:/var/lib/cni:rw \
-    --volume /var/lib/docker/:/var/lib/docker:rw \
-    --volume /var/lib/kubelet/:/var/lib/kubelet:shared \
-    --volume /var/log:/var/log:shared \
-    --volume /etc/kubernetes:/etc/kubernetes:ro \
-    --volume /etc/cni/net.d:/etc/cni/net.d:rw \
-    --volume /etc/resolv.conf:/etc/resolv.conf:ro \
-    --volume /opt/cni/bin:/opt/cni/bin:rw \
-    --volume /var/run/calico:/var/run/calico:rw \
-    --volume /var/lib/calico:/var/lib/calico:rw \
-    --volume /usr/sbin/modprobe:/usr/sbin/modprobe:rw \
-    --volume /lib/modules:/lib/modules:rw \
-    --entrypoint /usr/local/bin/kubelet \
-  "${kubelet_image_url}:${kubelet_image_tag}" \
-    --allow-privileged \
-    ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
-    --cni-bin-dir=/opt/cni/bin \
-    --cni-conf-dir=/etc/cni/net.d \
-    --config=/etc/kubernetes/config/worker-kubelet-conf.yaml \
-    --container-runtime=docker \
-    --exit-on-lock-contention \
-    --kubeconfig=/var/lib/kubelet/kubeconfig \
-    --network-plugin=cni \
-    --node-labels=role=${role} \
-    --lock-file=/var/run/lock/kubelet.lock \
-    --v=0
+ExecStart=${kubelet_binary_path} \
+  --allow-privileged \
+  ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
+  --cni-bin-dir=/opt/cni/bin \
+  --cni-conf-dir=/etc/cni/net.d \
+  --config=/etc/kubernetes/config/worker-kubelet-conf.yaml \
+  --container-runtime=docker \
+  --exit-on-lock-contention \
+  --kubeconfig=/var/lib/kubelet/kubeconfig \
+  --network-plugin=cni \
+  --node-labels=role=${role} \
+  --lock-file=/var/run/lock/kubelet.lock \
+  --v=0
 Restart=always
 RestartSec=10
 [Install]

diff --git a/worker.tf b/worker.tf
@@ -34,10 +34,9 @@ data "template_file" "worker-kubelet" {
   template = file("${path.module}/resources/worker-kubelet.service")
 
   vars = {
-    kubelet_image_url = var.hyperkube_image_url
-    kubelet_image_tag = var.hyperkube_image_tag
-    cloud_provider    = var.cloud_provider
-    role              = "worker"
+    kubelet_binary_path = "/opt/bin/kubelet"
+    cloud_provider      = var.cloud_provider
+    role                = "worker"
   }
 }
 
@@ -146,31 +145,32 @@ data "ignition_file" "prometheus-ro-rootfs" {
 // data.ignition_file.worker-prom-machine-role.id,
 data "ignition_config" "worker" {
   files = concat(
-      [
-        data.ignition_file.cfssl.id,
-        data.ignition_file.cfssljson.id,
-        data.ignition_file.cfssl-client-config.id,
-        data.ignition_file.worker-cfssl-new-cert.id,
-        data.ignition_file.worker-kubeconfig.id,
-        data.ignition_file.worker-sysctl-vm.id,
-        data.ignition_file.worker-kubelet-conf.id,
-        data.ignition_file.prometheus-ro-rootfs.id,
-      ],
-      var.worker_additional_files
-    )
+    [
+      data.ignition_file.cfssl.id,
+      data.ignition_file.cfssljson.id,
+      data.ignition_file.cfssl-client-config.id,
+      data.ignition_file.worker-cfssl-new-cert.id,
+      data.ignition_file.kubelet.id,
+      data.ignition_file.worker-kubeconfig.id,
+      data.ignition_file.worker-sysctl-vm.id,
+      data.ignition_file.worker-kubelet-conf.id,
+      data.ignition_file.prometheus-ro-rootfs.id,
+    ],
+    var.worker_additional_files
+  )
 
   systemd = concat(
-      [
-        data.ignition_systemd_unit.update-engine.id,
-        data.ignition_systemd_unit.locksmithd_worker.id,
-        data.ignition_systemd_unit.docker-opts-dropin.id,
-        data.ignition_systemd_unit.worker-kubelet.id,
-        data.ignition_systemd_unit.prometheus-tmpfs-dir.id,
-        data.ignition_systemd_unit.prometheus-machine-role.id,
-        data.ignition_systemd_unit.prometheus-ro-rootfs.id,
-        data.ignition_systemd_unit.prometheus-ro-rootfs-timer.id,
-      ],
-      module.kubelet-restarter.systemd_units,
-      var.worker_additional_systemd_units
-    )
+    [
+      data.ignition_systemd_unit.update-engine.id,
+      data.ignition_systemd_unit.locksmithd_worker.id,
+      data.ignition_systemd_unit.docker-opts-dropin.id,
+      data.ignition_systemd_unit.worker-kubelet.id,
+      data.ignition_systemd_unit.prometheus-tmpfs-dir.id,
+      data.ignition_systemd_unit.prometheus-machine-role.id,
+      data.ignition_systemd_unit.prometheus-ro-rootfs.id,
+      data.ignition_systemd_unit.prometheus-ro-rootfs-timer.id,
+    ],
+    module.kubelet-restarter.systemd_units,
+    var.worker_additional_systemd_units
+  )
 }