Merge branch 'feature/etcd-hardening' into develop

pblaas · Sep 22, 2017 · f20c255 · f20c255
2 parents 0304715 + 968e2b3
commit f20c255
Show file tree

Hide file tree

Showing 6 changed files with 222 additions and 25 deletions.
diff --git a/add_node.sh b/add_node.sh
@@ -11,7 +11,7 @@ if [ ! -f config.env ]; then
 fi
 . config.env
 
-if [ ! $1 ]; then 
+if [ ! $1 ]; then
 	echo You need to provide one or more ip adresses.
 	echo e.g $0 192.168.10.12
 	exit 1
@@ -29,6 +29,12 @@ WORKER_IP=${i} openssl req -new -key ${i}-worker-key.pem -out ${i}-worker.csr -s
 WORKER_IP=${i} openssl x509 -req -in ${i}-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
 done
 
+for i in $1; do
+openssl genrsa -out ${i}-etcd-worker-key.pem 2048
+WORKER_IP=${i} openssl req -new -key ${i}-etcd-worker-key.pem -out ${i}-etcd-worker.csr -subj "/CN=${i}" -config ../template/worker-openssl.cnf
+WORKER_IP=${i} openssl x509 -req -in ${i}-etcd-worker.csr -CA etcd-ca.pem -CAkey etcd-ca-key.pem -CAcreateserial -out ${i}-etcd-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
+done
+
 
 #gzip base64 encode files to store in the cloud init files.
 CAKEY=$(cat ca-key.pem | gzip | base64 -w0)
@@ -39,10 +45,17 @@ APISERVER=$(cat apiserver.pem | gzip | base64 -w0)
 for i in $1; do
 	j=$i-worker-key.pem
 	k=$i-worker.pem
+	l=$i-etcd-worker-key.pem
+	m=$i-etcd-worker.pem
 	WORKERKEY=$(cat $j | gzip | base64 -w0)
 	WORKER=$(cat $k | gzip | base64 -w0)
+	ETCDWORKERKEY=$(cat $l | gzip | base64 -w0)
+	ETCDWORKER=$(cat $m | gzip | base64 -w0)
 	echo WORKERKEY_$i:$WORKERKEY >> index.txt
 	echo WORKER_$i:$WORKER >> index.txt
+	echo ETCDWORKERKEY_$i:$ETCDWORKERKEY >> index.txt
+	echo ETCDWORKER_$i:$ETCDWORKER >> index.txt
+
 done
 
 #genereate the worker yamls from the worker.yaml template
@@ -58,9 +71,12 @@ sed -e "s,WORKER_IP,$i,g" \
 -e "s,USER_CORE_SSHKEY2,${USER_CORE_KEY2}," \
 -e "s,USER_CORE_PASSWORD,${HASHED_USER_CORE_PASSWORD},g" \
 -e "s,K8S_VER,$K8S_VER,g" \
--e "s,CACERT,$CACERT,g" \
--e "s,WORKERKEY,`cat index.txt|grep WORKERKEY_$i|cut -d: -f2`,g" \
--e "s,WORKER,`cat index.txt|grep WORKER_$i|cut -d: -f2`,g" \
+-e "s,\<CACERT\>,$CACERT,g" \
+-e "s,\<WORKERKEY\>,`cat index.txt|grep -w WORKERKEY_$i|cut -d: -f2`,g" \
+-e "s,\<WORKER\>,`cat index.txt|grep -w WORKER_$i|cut -d: -f2`,g" \
+-e "s,ETCDCACERT,`cat index.txt|grep -w ETCDCACERT|cut -d: -f2`,g" \
+-e "s,ETCDWORKERKEY,`cat index.txt|grep -w ETCDWORKERKEY_$i|cut -d: -f2`,g" \
+-e "s,ETCDWORKER,`cat index.txt|grep -w ETCDWORKER_$i|cut -d: -f2`,g" \
 ../template/worker_proxy.yaml > node_$i.yaml
 echo Generated: node_$i.yaml
 done

diff --git a/config.env.sample b/config.env.sample
@@ -6,7 +6,7 @@ WORKER_GW=192.168.2.1
 WORKER_IP1=192.168.2.20
 WORKER_IP2=192.168.2.21
 WORKER_HOSTS=(192.168.2.20 192.168.2.21)
-K8S_VER=v1.7.2_coreos.0
+K8S_VER=v1.7.6_coreos.0
 K8S_SERVICE_IP=10.3.0.1
 DNSSERVER=8.8.8.8
 CLUSTER_DNS=10.3.0.10

diff --git a/create_cloudinit.sh b/create_cloudinit.sh
@@ -39,20 +39,38 @@ echo DISCOVERY_ID:$DISCOVERY_ID >> index.txt
 openssl genrsa -out ca-key.pem 2048
 openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"
 
+#create etcd CA
+openssl genrsa -out etcd-ca-key.pem 2048
+openssl req -x509 -new -nodes -key etcd-ca-key.pem -days 10000 -out etcd-ca.pem -subj "/CN=etcd-ca"
+
+
 sed -e s/K8S_SERVICE_IP/$K8S_SERVICE_IP/ -e s/MASTER_HOST_IP/$MASTER_HOST_IP/ -e s/FLOATING_IP/$FLOATING_IP/ ../template/openssl.cnf > openssl.cnf
 
 #create API certs
 openssl genrsa -out apiserver-key.pem 2048
 openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf
 openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
 
+#create ETCD-API-certs
+openssl genrsa -out etcd-apiserver-key.pem 2048
+openssl req -new -key etcd-apiserver-key.pem -out etcd-apiserver.csr -subj "/CN=etcd-kube-apiserver" -config openssl.cnf
+openssl x509 -req -in etcd-apiserver.csr -CA etcd-ca.pem -CAkey etcd-ca-key.pem -CAcreateserial -out etcd-apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
+
 #create worker certs
 for i in ${WORKER_HOSTS[@]}; do
 openssl genrsa -out ${i}-worker-key.pem 2048
 WORKER_IP=${i} openssl req -new -key ${i}-worker-key.pem -out ${i}-worker.csr -subj "/CN=${i}" -config ../template/worker-openssl.cnf
 WORKER_IP=${i} openssl x509 -req -in ${i}-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
 done
 
+#create ETCD-worker certs
+for i in ${WORKER_HOSTS[@]}; do
+openssl genrsa -out ${i}-etcd-worker-key.pem 2048
+WORKER_IP=${i} openssl req -new -key ${i}-etcd-worker-key.pem -out ${i}-etcd-worker.csr -subj "/CN=${i}" -config ../template/worker-openssl.cnf
+WORKER_IP=${i} openssl x509 -req -in ${i}-etcd-worker.csr -CA etcd-ca.pem -CAkey etcd-ca-key.pem -CAcreateserial -out ${i}-etcd-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
+done
+
+
 #create admin certs
 openssl genrsa -out admin-key.pem 2048
 openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"
@@ -71,16 +89,27 @@ openssl x509 -req -in demouser.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial
 #gzip base64 encode files to store in the cloud init files.
 CAKEY=$(cat ca-key.pem | gzip | base64 -w0)
 CACERT=$(cat ca.pem | gzip | base64 -w0)
+ETCDCAKEY=$(cat etcd-ca-key.pem | gzip | base64 -w0)
+ETCDCACERT=$(cat etcd-ca.pem | gzip | base64 -w0)
 APISERVERKEY=$(cat apiserver-key.pem | gzip | base64 -w0)
 APISERVER=$(cat apiserver.pem | gzip | base64 -w0)
+ETCDAPISERVERKEY=$(cat etcd-apiserver-key.pem | gzip | base64 -w0)
+ETCDAPISERVER=$(cat etcd-apiserver.pem | gzip | base64 -w0)
+
 
 for i in ${WORKER_HOSTS[@]}; do
 	j=$i-worker-key.pem
 	k=$i-worker.pem
+	l=$i-etcd-worker-key.pem
+	m=$i-etcd-worker.pem
 	WORKERKEY=$(cat $j | gzip | base64 -w0)
 	WORKER=$(cat $k | gzip | base64 -w0)
+	ETCDWORKERKEY=$(cat $l | gzip | base64 -w0)
+	ETCDWORKER=$(cat $m | gzip | base64 -w0)
 	echo WORKERKEY_$i:$WORKERKEY >> index.txt
 	echo WORKER_$i:$WORKER >> index.txt
+	echo ETCDWORKERKEY_$i:$ETCDWORKERKEY >> index.txt
+	echo ETCDWORKER_$i:$ETCDWORKER >> index.txt
 done
 
 ADMINKEY=`cat admin-key.pem | gzip | base64 -w0`
@@ -89,6 +118,8 @@ ADMIN=`cat admin.pem | gzip | base64 -w0`
 #create indexfile with hashes
 echo CAKEY:$CAKEY >> index.txt
 echo CACERT:$CACERT >> index.txt
+echo ETCDCAKEY:$ETCDCAKEY >> index.txt
+echo ETCDCACERT:$ETCDCACERT >> index.txt
 echo APISERVERKEY:$APISERVERKEY >> index.txt
 echo APISERVER:$APISERVER >> index.txt
 echo ADMINKEY:$ADMINKEY >> index.txt
@@ -110,9 +141,12 @@ sed -e "s,MASTER_HOST_FQDN,$MASTER_HOST_FQDN,g" \
 -e "s,USER_CORE_SSHKEY2,${USER_CORE_KEY2}," \
 -e "s,USER_CORE_PASSWORD,$HASHED_USER_CORE_PASSWORD,g" \
 -e "s,K8S_VER,$K8S_VER,g" \
--e "s,CACERT,$CACERT,g" \
--e "s,APISERVERKEY,$APISERVERKEY,g" \
--e "s,APISERVER,$APISERVER,g" \
+-e "s,\<CACERT\>,$CACERT,g" \
+-e "s,\<APISERVERKEY\>,$APISERVERKEY,g" \
+-e "s,\<APISERVER\>,$APISERVER,g" \
+-e "s,ETCDCACERT,$ETCDCACERT,g" \
+-e "s,ETCDAPISERVERKEY,$ETCDAPISERVERKEY,g" \
+-e "s,ETCDAPISERVER,$ETCDAPISERVER,g" \
 ../template/controller.yaml > node_$MASTER_HOST_IP.yaml
 echo ----------------------
 echo Generated: Master: node_$MASTER_HOST_IP.yaml
@@ -130,9 +164,12 @@ sed -e "s,WORKER_IP,$i,g" \
 -e "s,USER_CORE_SSHKEY2,${USER_CORE_KEY2}," \
 -e "s,USER_CORE_PASSWORD,$HASHED_USER_CORE_PASSWORD,g" \
 -e "s,K8S_VER,$K8S_VER,g" \
--e "s,CACERT,$CACERT,g" \
--e "s,WORKERKEY,`cat index.txt|grep WORKERKEY_$i|cut -d: -f2`,g" \
--e "s,WORKER,`cat index.txt|grep WORKER_$i|cut -d: -f2`,g" \
+-e "s,\<CACERT\>,$CACERT,g" \
+-e "s,\<WORKERKEY\>,`cat index.txt|grep -w WORKERKEY_$i|cut -d: -f2`,g" \
+-e "s,\<WORKER\>,`cat index.txt|grep -w WORKER_$i|cut -d: -f2`,g" \
+-e "s,ETCDCACERT,$ETCDCACERT,g" \
+-e "s,ETCDWORKERKEY,`cat index.txt|grep -w ETCDWORKERKEY_$i|cut -d: -f2`,g" \
+-e "s,ETCDWORKER,`cat index.txt|grep -w ETCDWORKER_$i|cut -d: -f2`,g" \
 ../template/worker.yaml > node_$i.yaml
 echo Generated: Worker: node_$i.yaml
 done

diff --git a/template/controller.yaml b/template/controller.yaml
@@ -6,10 +6,18 @@ coreos:
     interface: MASTER_HOST_IP
   etcd2:
     discovery: https://discovery.etcd.io/DISCOVERY_ID
-    advertise-client-urls: http://MASTER_HOST_IP:2379
-    initial-advertise-peer-urls: http://MASTER_HOST_IP:2380
-    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
-    listen-peer-urls: http://MASTER_HOST_IP:2380
+    advertise-client-urls: https://MASTER_HOST_IP:2379
+    initial-advertise-peer-urls: https://MASTER_HOST_IP:2380
+    listen-client-urls: http://127.0.0.1:2379,https://MASTER_HOST_IP:2379
+    listen-peer-urls: https://MASTER_HOST_IP:2380
+    cert-file: /etc/kubernetes/ssl/etcd-apiserver.pem
+    key-file: /etc/kubernetes/ssl/etcd-apiserver-key.pem
+    trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
+    client-cert-auth: true
+    peer-cert-file: /etc/kubernetes/ssl/etcd-apiserver.pem
+    peer-key-file: /etc/kubernetes/ssl/etcd-apiserver-key.pem
+    peer-trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
+    peer-client-cert-auth: true
   fleet:
     metadata: "role=node"
   units:
@@ -46,7 +54,7 @@ coreos:
             Requires=etcd2.service
             After=etcd2.service
             [Service]
-            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
+            ExecStartPre=/usr/bin/etcdctl --cert-file=/etc/kubernetes/ssl/etcd-apiserver.pem --key-file=/etc/kubernetes/ssl/etcd-apiserver-key.pem --ca-file=/etc/kubernetes/ssl/etcd-ca.pem set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
             ExecStartPre=/usr/bin/ln -sf /etc/flannel/options.env /run/flannel/options.env
       command: start
     - name: kubelet.service
@@ -238,6 +246,9 @@ write_files:
     content: |
       FLANNELD_IFACE=MASTER_HOST_IP
       FLANNELD_ETCD_ENDPOINTS=ETCD_ENDPOINTS_URLS
+      FLANNELD_ETCD_KEYFILE=/etc/ssl/certs/etcd-apiserver-key.pem
+      FLANNELD_ETCD_CERTFILE=/etc/ssl/certs/etcd-apiserver.pem
+      FLANNELD_ETCD_CAFILE=/etc/ssl/certs/etcd-ca.pem
   - path: "/etc/kubernetes/manifests/kube-apiserver.yaml"
     permissions: "0644"
     owner: "root"
@@ -257,6 +268,9 @@ write_files:
           - apiserver
           - --bind-address=0.0.0.0
           - --etcd-servers=ETCD_ENDPOINTS_URLS
+          - --etcd-cafile=/etc/kubernetes/ssl/etcd-ca.pem
+          - --etcd-certfile=/etc/kubernetes/ssl/etcd-apiserver.pem
+          - --etcd-keyfile=/etc/kubernetes/ssl/etcd-apiserver-key.pem
           - --allow-privileged=true
           - --storage-backend=etcd2
           - --service-cluster-ip-range=SERVICE_CLUSTER_IP_RANGE
@@ -421,6 +435,42 @@ write_files:
     owner: "root"
     content: |
       CACERT
+  - path: "/etc/kubernetes/ssl/etcd-apiserver-key.pem"
+    permissions: "0644"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDAPISERVERKEY
+  - path: "/etc/kubernetes/ssl/etcd-apiserver.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDAPISERVER
+  - path: "/etc/kubernetes/ssl/etcd-ca.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDCACERT
+  - path: "/etc/ssl/certs/etcd-apiserver-key.pem"
+    permissions: "0644"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDAPISERVERKEY
+  - path: "/etc/ssl/certs/etcd-apiserver.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDAPISERVER
+  - path: "/etc/ssl/certs/etcd-ca.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDCACERT
   - path: /etc/motd.d/k8s.conf
     owner: "root"
     permissions: "0644"

diff --git a/template/worker.yaml b/template/worker.yaml
@@ -6,10 +6,18 @@ coreos:
     interface: WORKER_IP
   etcd2:
     discovery: https://discovery.etcd.io/DISCOVERY_ID
-    advertise-client-urls: http://WORKER_IP:2379
-    initial-advertise-peer-urls: http://WORKER_IP:2380
-    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
-    listen-peer-urls: http://WORKER_IP:2380
+    advertise-client-urls: https://WORKER_IP:2379
+    initial-advertise-peer-urls: https://WORKER_IP:2380
+    listen-client-urls: http://127.0.0.1:2379,https://WORKER_IP:2379
+    listen-peer-urls: https://WORKER_IP:2380
+    cert-file: /etc/kubernetes/ssl/etcd-worker.pem
+    key-file: /etc/kubernetes/ssl/etcd-worker-key.pem
+    trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
+    client-cert-auth: true
+    peer-cert-file: /etc/kubernetes/ssl/etcd-worker.pem
+    peer-key-file: /etc/kubernetes/ssl/etcd-worker-key.pem
+    peer-trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
+    peer-client-cert-auth: true
   fleet:
     metadata: "role=node"
   units:
@@ -46,7 +54,7 @@ coreos:
             Requires=etcd2.service
             After=etcd2.service
             [Service]
-            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
+            ExecStartPre=/usr/bin/etcdctl -cert-file=/etc/kubernetes/ssl/etcd-worker.pem --key-file=/etc/kubernetes/ssl/etcd-worker-key.pem --ca-file=/etc/kubernetes/ssl/etcd-ca.pem set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
             ExecStartPre=/usr/bin/ln -sf /etc/flannel/options.env /run/flannel/options.env
       command: start
     - name: kubelet.service
@@ -237,6 +245,9 @@ write_files:
     content: |
       FLANNELD_IFACE=WORKER_IP
       FLANNELD_ETCD_ENDPOINTS=ETCD_ENDPOINTS_URLS
+      FLANNELD_ETCD_KEYFILE=/etc/ssl/certs/etcd-worker-key.pem
+      FLANNELD_ETCD_CERTFILE=/etc/ssl/certs/etcd-worker.pem
+      FLANNELD_ETCD_CAFILE=/etc/ssl/certs/etcd-ca.pem
   - path: "/etc/kubernetes/manifests/kube-proxy.yaml"
     permissions: "0644"
     owner: "root"
@@ -318,6 +329,42 @@ write_files:
     owner: "root"
     content: |
       WORKER
+  - path: "/etc/kubernetes/ssl/etcd-ca.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDCACERT
+  - path: "/etc/kubernetes/ssl/etcd-worker-key.pem"
+    permissions: "0644"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDWORKERKEY
+  - path: "/etc/kubernetes/ssl/etcd-worker.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDWORKER
+  - path: "/etc/ssl/certs/etcd-ca.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDCACERT
+  - path: "/etc/ssl/certs/etcd-worker-key.pem"
+    permissions: "0644"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDWORKERKEY
+  - path: "/etc/ssl/certs/etcd-worker.pem"
+    permissions: "0664"
+    encoding: "gzip+base64"
+    owner: "root"
+    content: |
+      ETCDWORKER
   - path: /etc/motd.d/k8s.conf
     owner: "root"
     permissions: "0644"