From b05f9895606795ac745a0f07afef0698400f2815 Mon Sep 17 00:00:00 2001 From: Foivos Filippopoulos Date: Tue, 16 Jul 2019 19:05:46 +0100 Subject: [PATCH 1/2] Run kubelet binary directly instead of containerized --- common.tf | 31 ++++++++++++ master.tf | 73 ++++++++++++++-------------- resources/fetch-kubelet.service.tmpl | 11 +++++ resources/fetch-kubelet.tmpl | 7 +++ resources/master-kubelet.service | 51 ++++++------------- resources/worker-kubelet.service | 51 ++++++------------- variables.tf | 5 ++ worker.tf | 59 +++++++++++----------- 8 files changed, 150 insertions(+), 138 deletions(-) create mode 100644 resources/fetch-kubelet.service.tmpl create mode 100644 resources/fetch-kubelet.tmpl diff --git a/common.tf b/common.tf index 205fa76..89ef0c5 100644 --- a/common.tf +++ b/common.tf @@ -65,3 +65,34 @@ data "ignition_file" "format-and-mount" { content = file("${path.module}/resources/format-and-mount") } } + +data "template_file" "fetch-kubelet-script" { + template = file("${path.module}/resources/fetch-kubelet.tmpl") + + vars = { + kubelet_binary_version = var.kubelet_binary_version + } +} + +data "ignition_file" "fetch-kubelet-script" { + mode = 493 + filesystem = "root" + path = "/opt/bin/fetch-kubelet" + + content { + content = data.template_file.fetch-kubelet-script.rendered + } +} + +data "template_file" "fetch-kubelet-service" { + template = file("${path.module}/resources/fetch-kubelet.service.tmpl") + + vars = { + fetch_script_path = "/opt/bin/fetch-kubelet" + } +} + +data "ignition_systemd_unit" "fetch-kubelet-service" { + name = "fetch-kubelet.service" + content = data.template_file.fetch-kubelet-service.rendered +} diff --git a/master.tf b/master.tf index 7104a80..7777ab4 100644 --- a/master.tf +++ b/master.tf @@ -178,9 +178,8 @@ data "template_file" "master-kubelet" { template = file("${path.module}/resources/master-kubelet.service") vars = { - kubelet_image_url = var.hyperkube_image_url - kubelet_image_tag = var.hyperkube_image_tag - cloud_provider = var.cloud_provider + kubelet_binary_path = "/opt/bin/kubelet" + cloud_provider = var.cloud_provider } } @@ -390,39 +389,41 @@ locals { data "ignition_config" "master" { files = concat( - [ - data.ignition_file.audit-policy.id, - data.ignition_file.cfssl.id, - data.ignition_file.cfssljson.id, - data.ignition_file.cfssl-client-config.id, - data.ignition_file.master-cfssl-new-node-cert.id, - data.ignition_file.master-cfssl-new-apiserver-cert.id, - data.ignition_file.master-cfssl-new-apiserver-kubelet-client-cert.id, - data.ignition_file.master-cfssl-new-scheduler-cert.id, - data.ignition_file.master-cfssl-new-controller-manager-cert.id, - data.ignition_file.master-cfssl-keys-and-certs-get.id, - data.ignition_file.master-prom-machine-role.id, - data.ignition_file.scheduler-kubeconfig.id, - data.ignition_file.controller-manager-kubeconfig.id, - data.ignition_file.kubelet-kubeconfig.id, - data.ignition_file.kube-apiserver.id, - data.ignition_file.kube-scheduler.id, - data.ignition_file.kube-scheduler-config.id, - data.ignition_file.kube-controller-manager.id, - data.ignition_file.master-kubelet-conf.id, - ], - var.master_additional_files, - [local.kube_controller_additional_config] - ) + [ + data.ignition_file.audit-policy.id, + data.ignition_file.cfssl.id, + data.ignition_file.cfssljson.id, + data.ignition_file.cfssl-client-config.id, + data.ignition_file.master-cfssl-new-node-cert.id, + data.ignition_file.master-cfssl-new-apiserver-cert.id, + data.ignition_file.master-cfssl-new-apiserver-kubelet-client-cert.id, + data.ignition_file.master-cfssl-new-scheduler-cert.id, + data.ignition_file.master-cfssl-new-controller-manager-cert.id, + data.ignition_file.master-cfssl-keys-and-certs-get.id, + data.ignition_file.master-prom-machine-role.id, + data.ignition_file.scheduler-kubeconfig.id, + data.ignition_file.controller-manager-kubeconfig.id, + data.ignition_file.kubelet-kubeconfig.id, + data.ignition_file.kube-apiserver.id, + data.ignition_file.kube-scheduler.id, + data.ignition_file.kube-scheduler-config.id, + data.ignition_file.kube-controller-manager.id, + data.ignition_file.fetch-kubelet-script.id, + data.ignition_file.master-kubelet-conf.id, + ], + var.master_additional_files, + [local.kube_controller_additional_config] + ) systemd = concat( - [ - data.ignition_systemd_unit.update-engine.id, - data.ignition_systemd_unit.locksmithd_master.id, - data.ignition_systemd_unit.docker-opts-dropin.id, - data.ignition_systemd_unit.master-kubelet.id, - ], - module.kubelet-restarter.systemd_units, - var.master_additional_systemd_units - ) + [ + data.ignition_systemd_unit.update-engine.id, + data.ignition_systemd_unit.locksmithd_master.id, + data.ignition_systemd_unit.docker-opts-dropin.id, + data.ignition_systemd_unit.fetch-kubelet-service.id, + data.ignition_systemd_unit.master-kubelet.id, + ], + module.kubelet-restarter.systemd_units, + var.master_additional_systemd_units + ) } diff --git a/resources/fetch-kubelet.service.tmpl b/resources/fetch-kubelet.service.tmpl new file mode 100644 index 0000000..eac2d03 --- /dev/null +++ b/resources/fetch-kubelet.service.tmpl @@ -0,0 +1,11 @@ +[Unit] +Description=Fetches kubelet binary +Wants=network-online.target +[Service] +Type=oneshot +RemainAfterExit=yes +# oneshot systemd services do not support `Restart` +# https://github.com/systemd/systemd/issues/2582 +ExecStart=/bin/sh -c 'while ! ${fetch_script_path}; do sleep 1; done' +[Install] +WantedBy=multi-user.target diff --git a/resources/fetch-kubelet.tmpl b/resources/fetch-kubelet.tmpl new file mode 100644 index 0000000..24a2641 --- /dev/null +++ b/resources/fetch-kubelet.tmpl @@ -0,0 +1,7 @@ +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/?origin_team=T0HR00WDA#installing-kubeadm-kubelet-and-kubectl +RELEASE=${kubelet_binary_version} + +mkdir -p /opt/bin +cd /opt/bin +curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/$${RELEASE}/bin/linux/amd64/kubelet +chmod +x kubelet diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service index 120a383..38e38a5 100644 --- a/resources/master-kubelet.service +++ b/resources/master-kubelet.service @@ -1,8 +1,7 @@ -# https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service [Unit] Description=Kubernetes Kubelet Requires=docker.service -After=docker.service +After=docker.service,fetch-kubelet.service [Service] ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/usr/bin/mkdir -p /var/log/containers @@ -25,40 +24,20 @@ ExecStartPre=/opt/bin/cfssl-new-scheduler-cert ExecStartPre=/opt/bin/cfssl-new-controller-manager-cert ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-controller-manager' | awk '{ print $1; }')" ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-apiserver' | awk '{ print $1; }')" -ExecStart=/usr/bin/docker \ - run \ - --rm \ - --net host \ - --pid host \ - --privileged \ - --volume /dev:/dev:rw \ - --volume /sys:/sys:ro \ - --volume /var/run:/var/run:rw \ - --volume /var/lib/cni/:/var/lib/cni:rw \ - --volume /var/lib/docker/:/var/lib/docker:rw \ - --volume /var/lib/kubelet/:/var/lib/kubelet:shared \ - --volume /var/log:/var/log:shared \ - --volume /etc/kubernetes:/etc/kubernetes:ro \ - --volume /etc/cni/net.d:/etc/cni/net.d:rw \ - --volume /etc/resolv.conf:/etc/resolv.conf:ro \ - --volume /opt/cni/bin:/opt/cni/bin:rw \ - --volume /var/run/calico:/var/run/calico:rw \ - --volume /var/lib/calico:/var/lib/calico:rw \ - --entrypoint /usr/local/bin/kubelet \ - "${kubelet_image_url}:${kubelet_image_tag}" \ - --allow-privileged \ - --config=/etc/kubernetes/config/master-kubelet-conf.yaml \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --node-labels=role=master,node-role.kubernetes.io/master="" \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --container-runtime=docker \ - --network-plugin=cni \ - --cni-bin-dir=/opt/cni/bin \ - --cni-conf-dir=/etc/cni/net.d \ - ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --v=0 +ExecStart=${kubelet_binary_path} \ + --allow-privileged \ + --config=/etc/kubernetes/config/master-kubelet-conf.yaml \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --node-labels=role=master,node-role.kubernetes.io/master="" \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --container-runtime=docker \ + --network-plugin=cni \ + --cni-bin-dir=/opt/cni/bin \ + --cni-conf-dir=/etc/cni/net.d \ + ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --v=0 Restart=always RestartSec=10 [Install] diff --git a/resources/worker-kubelet.service b/resources/worker-kubelet.service index 967478f..fafe011 100644 --- a/resources/worker-kubelet.service +++ b/resources/worker-kubelet.service @@ -1,8 +1,7 @@ -# https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service [Unit] Description=Kubernetes Kubelet Requires=docker.service -After=docker.service +After=docker.service,fetch-kubelet.service [Service] ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/usr/bin/mkdir -p /var/log/containers @@ -18,41 +17,19 @@ ExecStartPre=/sbin/sysctl -w net.ipv4.tcp_retries2=8 # https://github.com/kubernetes/kubernetes/issues/69015 ExecStartPre=/sbin/sysctl -w fs.inotify.max_user_watches=524288 ExecStartPre=/opt/bin/cfssl-new-cert -ExecStart=/usr/bin/docker \ - run \ - --rm \ - --net host \ - --pid host \ - --privileged \ - --volume /dev:/dev:rw \ - --volume /sys:/sys:ro \ - --volume /var/run:/var/run:rw \ - --volume /var/lib/cni/:/var/lib/cni:rw \ - --volume /var/lib/docker/:/var/lib/docker:rw \ - --volume /var/lib/kubelet/:/var/lib/kubelet:shared \ - --volume /var/log:/var/log:shared \ - --volume /etc/kubernetes:/etc/kubernetes:ro \ - --volume /etc/cni/net.d:/etc/cni/net.d:rw \ - --volume /etc/resolv.conf:/etc/resolv.conf:ro \ - --volume /opt/cni/bin:/opt/cni/bin:rw \ - --volume /var/run/calico:/var/run/calico:rw \ - --volume /var/lib/calico:/var/lib/calico:rw \ - --volume /usr/sbin/modprobe:/usr/sbin/modprobe:rw \ - --volume /lib/modules:/lib/modules:rw \ - --entrypoint /usr/local/bin/kubelet \ - "${kubelet_image_url}:${kubelet_image_tag}" \ - --allow-privileged \ - ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \ - --cni-bin-dir=/opt/cni/bin \ - --cni-conf-dir=/etc/cni/net.d \ - --config=/etc/kubernetes/config/worker-kubelet-conf.yaml \ - --container-runtime=docker \ - --exit-on-lock-contention \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --network-plugin=cni \ - --node-labels=role=${role} \ - --lock-file=/var/run/lock/kubelet.lock \ - --v=0 +ExecStart=${kubelet_binary_path} \ + --allow-privileged \ + ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \ + --cni-bin-dir=/opt/cni/bin \ + --cni-conf-dir=/etc/cni/net.d \ + --config=/etc/kubernetes/config/worker-kubelet-conf.yaml \ + --container-runtime=docker \ + --exit-on-lock-contention \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --network-plugin=cni \ + --node-labels=role=${role} \ + --lock-file=/var/run/lock/kubelet.lock \ + --v=0 Restart=always RestartSec=10 [Install] diff --git a/variables.tf b/variables.tf index aaf6e8f..0761cd4 100644 --- a/variables.tf +++ b/variables.tf @@ -57,6 +57,11 @@ variable "hyperkube_image_tag" { default = "v1.14.2" } +variable "kubelet_binary_version" { + description = "kubelet binary version to fetch from https://storage.googleapis.com/kubernetes-release/release" + default = "v1.14.2" +} + variable "cluster_dns" { description = "List of DNS server IP addresses. Used by kubelet." type = list(string) diff --git a/worker.tf b/worker.tf index a93c0e5..29d201f 100644 --- a/worker.tf +++ b/worker.tf @@ -34,10 +34,9 @@ data "template_file" "worker-kubelet" { template = file("${path.module}/resources/worker-kubelet.service") vars = { - kubelet_image_url = var.hyperkube_image_url - kubelet_image_tag = var.hyperkube_image_tag - cloud_provider = var.cloud_provider - role = "worker" + kubelet_binary_path = "/opt/bin/kubelet" + cloud_provider = var.cloud_provider + role = "worker" } } @@ -146,31 +145,33 @@ data "ignition_file" "prometheus-ro-rootfs" { // data.ignition_file.worker-prom-machine-role.id, data "ignition_config" "worker" { files = concat( - [ - data.ignition_file.cfssl.id, - data.ignition_file.cfssljson.id, - data.ignition_file.cfssl-client-config.id, - data.ignition_file.worker-cfssl-new-cert.id, - data.ignition_file.worker-kubeconfig.id, - data.ignition_file.worker-sysctl-vm.id, - data.ignition_file.worker-kubelet-conf.id, - data.ignition_file.prometheus-ro-rootfs.id, - ], - var.worker_additional_files - ) + [ + data.ignition_file.cfssl.id, + data.ignition_file.cfssljson.id, + data.ignition_file.cfssl-client-config.id, + data.ignition_file.worker-cfssl-new-cert.id, + data.ignition_file.fetch-kubelet-script.id, + data.ignition_file.worker-kubeconfig.id, + data.ignition_file.worker-sysctl-vm.id, + data.ignition_file.worker-kubelet-conf.id, + data.ignition_file.prometheus-ro-rootfs.id, + ], + var.worker_additional_files + ) systemd = concat( - [ - data.ignition_systemd_unit.update-engine.id, - data.ignition_systemd_unit.locksmithd_worker.id, - data.ignition_systemd_unit.docker-opts-dropin.id, - data.ignition_systemd_unit.worker-kubelet.id, - data.ignition_systemd_unit.prometheus-tmpfs-dir.id, - data.ignition_systemd_unit.prometheus-machine-role.id, - data.ignition_systemd_unit.prometheus-ro-rootfs.id, - data.ignition_systemd_unit.prometheus-ro-rootfs-timer.id, - ], - module.kubelet-restarter.systemd_units, - var.worker_additional_systemd_units - ) + [ + data.ignition_systemd_unit.update-engine.id, + data.ignition_systemd_unit.locksmithd_worker.id, + data.ignition_systemd_unit.docker-opts-dropin.id, + data.ignition_systemd_unit.fetch-kubelet-service.id, + data.ignition_systemd_unit.worker-kubelet.id, + data.ignition_systemd_unit.prometheus-tmpfs-dir.id, + data.ignition_systemd_unit.prometheus-machine-role.id, + data.ignition_systemd_unit.prometheus-ro-rootfs.id, + data.ignition_systemd_unit.prometheus-ro-rootfs-timer.id, + ], + module.kubelet-restarter.systemd_units, + var.worker_additional_systemd_units + ) } From f55b7e57cf12d42b8ea14a3b58fe67ebedde38ff Mon Sep 17 00:00:00 2001 From: Foivos Filippopoulos Date: Wed, 17 Jul 2019 09:46:20 +0100 Subject: [PATCH 2/2] Fetch kubelet directly as ignition file --- common.tf | 29 ++++------------------------ master.tf | 3 +-- resources/fetch-kubelet.service.tmpl | 11 ----------- resources/fetch-kubelet.tmpl | 7 ------- resources/master-kubelet.service | 2 +- resources/worker-kubelet.service | 2 +- variables.tf | 5 ----- worker.tf | 3 +-- 8 files changed, 8 insertions(+), 54 deletions(-) delete mode 100644 resources/fetch-kubelet.service.tmpl delete mode 100644 resources/fetch-kubelet.tmpl diff --git a/common.tf b/common.tf index 89ef0c5..d418bbf 100644 --- a/common.tf +++ b/common.tf @@ -66,33 +66,12 @@ data "ignition_file" "format-and-mount" { } } -data "template_file" "fetch-kubelet-script" { - template = file("${path.module}/resources/fetch-kubelet.tmpl") - - vars = { - kubelet_binary_version = var.kubelet_binary_version - } -} - -data "ignition_file" "fetch-kubelet-script" { +data "ignition_file" "kubelet" { mode = 493 filesystem = "root" - path = "/opt/bin/fetch-kubelet" - - content { - content = data.template_file.fetch-kubelet-script.rendered - } -} + path = "/opt/bin/kubelet" -data "template_file" "fetch-kubelet-service" { - template = file("${path.module}/resources/fetch-kubelet.service.tmpl") - - vars = { - fetch_script_path = "/opt/bin/fetch-kubelet" + source { + source = "https://storage.googleapis.com/kubernetes-release/release/${var.hyperkube_image_tag}/bin/linux/amd64/kubelet" } } - -data "ignition_systemd_unit" "fetch-kubelet-service" { - name = "fetch-kubelet.service" - content = data.template_file.fetch-kubelet-service.rendered -} diff --git a/master.tf b/master.tf index 7777ab4..47ea6fe 100644 --- a/master.tf +++ b/master.tf @@ -408,7 +408,7 @@ data "ignition_config" "master" { data.ignition_file.kube-scheduler.id, data.ignition_file.kube-scheduler-config.id, data.ignition_file.kube-controller-manager.id, - data.ignition_file.fetch-kubelet-script.id, + data.ignition_file.kubelet.id, data.ignition_file.master-kubelet-conf.id, ], var.master_additional_files, @@ -420,7 +420,6 @@ data "ignition_config" "master" { data.ignition_systemd_unit.update-engine.id, data.ignition_systemd_unit.locksmithd_master.id, data.ignition_systemd_unit.docker-opts-dropin.id, - data.ignition_systemd_unit.fetch-kubelet-service.id, data.ignition_systemd_unit.master-kubelet.id, ], module.kubelet-restarter.systemd_units, diff --git a/resources/fetch-kubelet.service.tmpl b/resources/fetch-kubelet.service.tmpl deleted file mode 100644 index eac2d03..0000000 --- a/resources/fetch-kubelet.service.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Fetches kubelet binary -Wants=network-online.target -[Service] -Type=oneshot -RemainAfterExit=yes -# oneshot systemd services do not support `Restart` -# https://github.com/systemd/systemd/issues/2582 -ExecStart=/bin/sh -c 'while ! ${fetch_script_path}; do sleep 1; done' -[Install] -WantedBy=multi-user.target diff --git a/resources/fetch-kubelet.tmpl b/resources/fetch-kubelet.tmpl deleted file mode 100644 index 24a2641..0000000 --- a/resources/fetch-kubelet.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/?origin_team=T0HR00WDA#installing-kubeadm-kubelet-and-kubectl -RELEASE=${kubelet_binary_version} - -mkdir -p /opt/bin -cd /opt/bin -curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/$${RELEASE}/bin/linux/amd64/kubelet -chmod +x kubelet diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service index 38e38a5..4c8cf0e 100644 --- a/resources/master-kubelet.service +++ b/resources/master-kubelet.service @@ -1,7 +1,7 @@ [Unit] Description=Kubernetes Kubelet Requires=docker.service -After=docker.service,fetch-kubelet.service +After=docker.service [Service] ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/usr/bin/mkdir -p /var/log/containers diff --git a/resources/worker-kubelet.service b/resources/worker-kubelet.service index fafe011..6baa723 100644 --- a/resources/worker-kubelet.service +++ b/resources/worker-kubelet.service @@ -1,7 +1,7 @@ [Unit] Description=Kubernetes Kubelet Requires=docker.service -After=docker.service,fetch-kubelet.service +After=docker.service [Service] ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/usr/bin/mkdir -p /var/log/containers diff --git a/variables.tf b/variables.tf index 0761cd4..aaf6e8f 100644 --- a/variables.tf +++ b/variables.tf @@ -57,11 +57,6 @@ variable "hyperkube_image_tag" { default = "v1.14.2" } -variable "kubelet_binary_version" { - description = "kubelet binary version to fetch from https://storage.googleapis.com/kubernetes-release/release" - default = "v1.14.2" -} - variable "cluster_dns" { description = "List of DNS server IP addresses. Used by kubelet." type = list(string) diff --git a/worker.tf b/worker.tf index 29d201f..540a43c 100644 --- a/worker.tf +++ b/worker.tf @@ -150,7 +150,7 @@ data "ignition_config" "worker" { data.ignition_file.cfssljson.id, data.ignition_file.cfssl-client-config.id, data.ignition_file.worker-cfssl-new-cert.id, - data.ignition_file.fetch-kubelet-script.id, + data.ignition_file.kubelet.id, data.ignition_file.worker-kubeconfig.id, data.ignition_file.worker-sysctl-vm.id, data.ignition_file.worker-kubelet-conf.id, @@ -164,7 +164,6 @@ data "ignition_config" "worker" { data.ignition_systemd_unit.update-engine.id, data.ignition_systemd_unit.locksmithd_worker.id, data.ignition_systemd_unit.docker-opts-dropin.id, - data.ignition_systemd_unit.fetch-kubelet-service.id, data.ignition_systemd_unit.worker-kubelet.id, data.ignition_systemd_unit.prometheus-tmpfs-dir.id, data.ignition_systemd_unit.prometheus-machine-role.id,