Skip to content

Commit f20c255

Browse files
pblaaspblaas
committed
Merge branch 'feature/etcd-hardening' into develop
2 parents 0304715 + 968e2b3 commit f20c255

File tree

6 files changed

+222
-25
lines changed

6 files changed

+222
-25
lines changed

‎add_node.sh‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ if [ ! -f config.env ]; then
1111
fi
1212
. config.env
1313

14-
if [ ! $1 ]; then
14+
if [ ! $1 ]; then
1515
echo You need to provide one or more ip adresses.
1616
echo e.g $0 192.168.10.12
1717
exit 1
@@ -29,6 +29,12 @@ WORKER_IP=${i} openssl req -new -key ${i}-worker-key.pem -out ${i}-worker.csr -s
2929
WORKER_IP=${i} openssl x509 -req -in ${i}-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
3030
done
3131

32+
for i in $1; do
33+
openssl genrsa -out ${i}-etcd-worker-key.pem 2048
34+
WORKER_IP=${i} openssl req -new -key ${i}-etcd-worker-key.pem -out ${i}-etcd-worker.csr -subj "/CN=${i}" -config ../template/worker-openssl.cnf
35+
WORKER_IP=${i} openssl x509 -req -in ${i}-etcd-worker.csr -CA etcd-ca.pem -CAkey etcd-ca-key.pem -CAcreateserial -out ${i}-etcd-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
36+
done
37+
3238

3339
#gzip base64 encode files to store in the cloud init files.
3440
CAKEY=$(cat ca-key.pem | gzip | base64 -w0)
@@ -39,10 +45,17 @@ APISERVER=$(cat apiserver.pem | gzip | base64 -w0)
3945
for i in $1; do
4046
j=$i-worker-key.pem
4147
k=$i-worker.pem
48+
l=$i-etcd-worker-key.pem
49+
m=$i-etcd-worker.pem
4250
WORKERKEY=$(cat $j | gzip | base64 -w0)
4351
WORKER=$(cat $k | gzip | base64 -w0)
52+
ETCDWORKERKEY=$(cat $l | gzip | base64 -w0)
53+
ETCDWORKER=$(cat $m | gzip | base64 -w0)
4454
echo WORKERKEY_$i:$WORKERKEY >> index.txt
4555
echo WORKER_$i:$WORKER >> index.txt
56+
echo ETCDWORKERKEY_$i:$ETCDWORKERKEY >> index.txt
57+
echo ETCDWORKER_$i:$ETCDWORKER >> index.txt
58+
4659
done
4760

4861
#genereate the worker yamls from the worker.yaml template
@@ -58,9 +71,12 @@ sed -e "s,WORKER_IP,$i,g" \
5871
-e "s,USER_CORE_SSHKEY2,${USER_CORE_KEY2}," \
5972
-e "s,USER_CORE_PASSWORD,${HASHED_USER_CORE_PASSWORD},g" \
6073
-e "s,K8S_VER,$K8S_VER,g" \
61-
-e "s,CACERT,$CACERT,g" \
62-
-e "s,WORKERKEY,`cat index.txt|grep WORKERKEY_$i|cut -d: -f2`,g" \
63-
-e "s,WORKER,`cat index.txt|grep WORKER_$i|cut -d: -f2`,g" \
74+
-e "s,\<CACERT\>,$CACERT,g" \
75+
-e "s,\<WORKERKEY\>,`cat index.txt|grep -w WORKERKEY_$i|cut -d: -f2`,g" \
76+
-e "s,\<WORKER\>,`cat index.txt|grep -w WORKER_$i|cut -d: -f2`,g" \
77+
-e "s,ETCDCACERT,`cat index.txt|grep -w ETCDCACERT|cut -d: -f2`,g" \
78+
-e "s,ETCDWORKERKEY,`cat index.txt|grep -w ETCDWORKERKEY_$i|cut -d: -f2`,g" \
79+
-e "s,ETCDWORKER,`cat index.txt|grep -w ETCDWORKER_$i|cut -d: -f2`,g" \
6480
../template/worker_proxy.yaml > node_$i.yaml
6581
echo Generated: node_$i.yaml
6682
done

‎config.env.sample‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ WORKER_GW=192.168.2.1
66
WORKER_IP1=192.168.2.20
77
WORKER_IP2=192.168.2.21
88
WORKER_HOSTS=(192.168.2.20 192.168.2.21)
9-
K8S_VER=v1.7.2_coreos.0
9+
K8S_VER=v1.7.6_coreos.0
1010
K8S_SERVICE_IP=10.3.0.1
1111
DNSSERVER=8.8.8.8
1212
CLUSTER_DNS=10.3.0.10

‎create_cloudinit.sh‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,20 +39,38 @@ echo DISCOVERY_ID:$DISCOVERY_ID >> index.txt
3939
openssl genrsa -out ca-key.pem 2048
4040
openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"
4141

42+
#create etcd CA
43+
openssl genrsa -out etcd-ca-key.pem 2048
44+
openssl req -x509 -new -nodes -key etcd-ca-key.pem -days 10000 -out etcd-ca.pem -subj "/CN=etcd-ca"
45+
46+
4247
sed -e s/K8S_SERVICE_IP/$K8S_SERVICE_IP/ -e s/MASTER_HOST_IP/$MASTER_HOST_IP/ -e s/FLOATING_IP/$FLOATING_IP/ ../template/openssl.cnf > openssl.cnf
4348

4449
#create API certs
4550
openssl genrsa -out apiserver-key.pem 2048
4651
openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf
4752
openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
4853

54+
#create ETCD-API-certs
55+
openssl genrsa -out etcd-apiserver-key.pem 2048
56+
openssl req -new -key etcd-apiserver-key.pem -out etcd-apiserver.csr -subj "/CN=etcd-kube-apiserver" -config openssl.cnf
57+
openssl x509 -req -in etcd-apiserver.csr -CA etcd-ca.pem -CAkey etcd-ca-key.pem -CAcreateserial -out etcd-apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
58+
4959
#create worker certs
5060
for i in ${WORKER_HOSTS[@]}; do
5161
openssl genrsa -out ${i}-worker-key.pem 2048
5262
WORKER_IP=${i} openssl req -new -key ${i}-worker-key.pem -out ${i}-worker.csr -subj "/CN=${i}" -config ../template/worker-openssl.cnf
5363
WORKER_IP=${i} openssl x509 -req -in ${i}-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
5464
done
5565

66+
#create ETCD-worker certs
67+
for i in ${WORKER_HOSTS[@]}; do
68+
openssl genrsa -out ${i}-etcd-worker-key.pem 2048
69+
WORKER_IP=${i} openssl req -new -key ${i}-etcd-worker-key.pem -out ${i}-etcd-worker.csr -subj "/CN=${i}" -config ../template/worker-openssl.cnf
70+
WORKER_IP=${i} openssl x509 -req -in ${i}-etcd-worker.csr -CA etcd-ca.pem -CAkey etcd-ca-key.pem -CAcreateserial -out ${i}-etcd-worker.pem -days 365 -extensions v3_req -extfile ../template/worker-openssl.cnf
71+
done
72+
73+
5674
#create admin certs
5775
openssl genrsa -out admin-key.pem 2048
5876
openssl req -new -key admin-key.pem -out admin.csr -subj "/CN=kube-admin"
@@ -71,16 +89,27 @@ openssl x509 -req -in demouser.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial
7189
#gzip base64 encode files to store in the cloud init files.
7290
CAKEY=$(cat ca-key.pem | gzip | base64 -w0)
7391
CACERT=$(cat ca.pem | gzip | base64 -w0)
92+
ETCDCAKEY=$(cat etcd-ca-key.pem | gzip | base64 -w0)
93+
ETCDCACERT=$(cat etcd-ca.pem | gzip | base64 -w0)
7494
APISERVERKEY=$(cat apiserver-key.pem | gzip | base64 -w0)
7595
APISERVER=$(cat apiserver.pem | gzip | base64 -w0)
96+
ETCDAPISERVERKEY=$(cat etcd-apiserver-key.pem | gzip | base64 -w0)
97+
ETCDAPISERVER=$(cat etcd-apiserver.pem | gzip | base64 -w0)
98+
7699

77100
for i in ${WORKER_HOSTS[@]}; do
78101
j=$i-worker-key.pem
79102
k=$i-worker.pem
103+
l=$i-etcd-worker-key.pem
104+
m=$i-etcd-worker.pem
80105
WORKERKEY=$(cat $j | gzip | base64 -w0)
81106
WORKER=$(cat $k | gzip | base64 -w0)
107+
ETCDWORKERKEY=$(cat $l | gzip | base64 -w0)
108+
ETCDWORKER=$(cat $m | gzip | base64 -w0)
82109
echo WORKERKEY_$i:$WORKERKEY >> index.txt
83110
echo WORKER_$i:$WORKER >> index.txt
111+
echo ETCDWORKERKEY_$i:$ETCDWORKERKEY >> index.txt
112+
echo ETCDWORKER_$i:$ETCDWORKER >> index.txt
84113
done
85114

86115
ADMINKEY=`cat admin-key.pem | gzip | base64 -w0`
@@ -89,6 +118,8 @@ ADMIN=`cat admin.pem | gzip | base64 -w0`
89118
#create indexfile with hashes
90119
echo CAKEY:$CAKEY >> index.txt
91120
echo CACERT:$CACERT >> index.txt
121+
echo ETCDCAKEY:$ETCDCAKEY >> index.txt
122+
echo ETCDCACERT:$ETCDCACERT >> index.txt
92123
echo APISERVERKEY:$APISERVERKEY >> index.txt
93124
echo APISERVER:$APISERVER >> index.txt
94125
echo ADMINKEY:$ADMINKEY >> index.txt
@@ -110,9 +141,12 @@ sed -e "s,MASTER_HOST_FQDN,$MASTER_HOST_FQDN,g" \
110141
-e "s,USER_CORE_SSHKEY2,${USER_CORE_KEY2}," \
111142
-e "s,USER_CORE_PASSWORD,$HASHED_USER_CORE_PASSWORD,g" \
112143
-e "s,K8S_VER,$K8S_VER,g" \
113-
-e "s,CACERT,$CACERT,g" \
114-
-e "s,APISERVERKEY,$APISERVERKEY,g" \
115-
-e "s,APISERVER,$APISERVER,g" \
144+
-e "s,\<CACERT\>,$CACERT,g" \
145+
-e "s,\<APISERVERKEY\>,$APISERVERKEY,g" \
146+
-e "s,\<APISERVER\>,$APISERVER,g" \
147+
-e "s,ETCDCACERT,$ETCDCACERT,g" \
148+
-e "s,ETCDAPISERVERKEY,$ETCDAPISERVERKEY,g" \
149+
-e "s,ETCDAPISERVER,$ETCDAPISERVER,g" \
116150
../template/controller.yaml > node_$MASTER_HOST_IP.yaml
117151
echo ----------------------
118152
echo Generated: Master: node_$MASTER_HOST_IP.yaml
@@ -130,9 +164,12 @@ sed -e "s,WORKER_IP,$i,g" \
130164
-e "s,USER_CORE_SSHKEY2,${USER_CORE_KEY2}," \
131165
-e "s,USER_CORE_PASSWORD,$HASHED_USER_CORE_PASSWORD,g" \
132166
-e "s,K8S_VER,$K8S_VER,g" \
133-
-e "s,CACERT,$CACERT,g" \
134-
-e "s,WORKERKEY,`cat index.txt|grep WORKERKEY_$i|cut -d: -f2`,g" \
135-
-e "s,WORKER,`cat index.txt|grep WORKER_$i|cut -d: -f2`,g" \
167+
-e "s,\<CACERT\>,$CACERT,g" \
168+
-e "s,\<WORKERKEY\>,`cat index.txt|grep -w WORKERKEY_$i|cut -d: -f2`,g" \
169+
-e "s,\<WORKER\>,`cat index.txt|grep -w WORKER_$i|cut -d: -f2`,g" \
170+
-e "s,ETCDCACERT,$ETCDCACERT,g" \
171+
-e "s,ETCDWORKERKEY,`cat index.txt|grep -w ETCDWORKERKEY_$i|cut -d: -f2`,g" \
172+
-e "s,ETCDWORKER,`cat index.txt|grep -w ETCDWORKER_$i|cut -d: -f2`,g" \
136173
../template/worker.yaml > node_$i.yaml
137174
echo Generated: Worker: node_$i.yaml
138175
done

‎template/controller.yaml‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,18 @@ coreos:
66
interface: MASTER_HOST_IP
77
etcd2:
88
discovery: https://discovery.etcd.io/DISCOVERY_ID
9-
advertise-client-urls: http://MASTER_HOST_IP:2379
10-
initial-advertise-peer-urls: http://MASTER_HOST_IP:2380
11-
listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
12-
listen-peer-urls: http://MASTER_HOST_IP:2380
9+
advertise-client-urls: https://MASTER_HOST_IP:2379
10+
initial-advertise-peer-urls: https://MASTER_HOST_IP:2380
11+
listen-client-urls: http://127.0.0.1:2379,https://MASTER_HOST_IP:2379
12+
listen-peer-urls: https://MASTER_HOST_IP:2380
13+
cert-file: /etc/kubernetes/ssl/etcd-apiserver.pem
14+
key-file: /etc/kubernetes/ssl/etcd-apiserver-key.pem
15+
trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
16+
client-cert-auth: true
17+
peer-cert-file: /etc/kubernetes/ssl/etcd-apiserver.pem
18+
peer-key-file: /etc/kubernetes/ssl/etcd-apiserver-key.pem
19+
peer-trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
20+
peer-client-cert-auth: true
1321
fleet:
1422
metadata: "role=node"
1523
units:
@@ -46,7 +54,7 @@ coreos:
4654
Requires=etcd2.service
4755
After=etcd2.service
4856
[Service]
49-
ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
57+
ExecStartPre=/usr/bin/etcdctl --cert-file=/etc/kubernetes/ssl/etcd-apiserver.pem --key-file=/etc/kubernetes/ssl/etcd-apiserver-key.pem --ca-file=/etc/kubernetes/ssl/etcd-ca.pem set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
5058
ExecStartPre=/usr/bin/ln -sf /etc/flannel/options.env /run/flannel/options.env
5159
command: start
5260
- name: kubelet.service
@@ -238,6 +246,9 @@ write_files:
238246
content: |
239247
FLANNELD_IFACE=MASTER_HOST_IP
240248
FLANNELD_ETCD_ENDPOINTS=ETCD_ENDPOINTS_URLS
249+
FLANNELD_ETCD_KEYFILE=/etc/ssl/certs/etcd-apiserver-key.pem
250+
FLANNELD_ETCD_CERTFILE=/etc/ssl/certs/etcd-apiserver.pem
251+
FLANNELD_ETCD_CAFILE=/etc/ssl/certs/etcd-ca.pem
241252
- path: "/etc/kubernetes/manifests/kube-apiserver.yaml"
242253
permissions: "0644"
243254
owner: "root"
@@ -257,6 +268,9 @@ write_files:
257268
- apiserver
258269
- --bind-address=0.0.0.0
259270
- --etcd-servers=ETCD_ENDPOINTS_URLS
271+
- --etcd-cafile=/etc/kubernetes/ssl/etcd-ca.pem
272+
- --etcd-certfile=/etc/kubernetes/ssl/etcd-apiserver.pem
273+
- --etcd-keyfile=/etc/kubernetes/ssl/etcd-apiserver-key.pem
260274
- --allow-privileged=true
261275
- --storage-backend=etcd2
262276
- --service-cluster-ip-range=SERVICE_CLUSTER_IP_RANGE
@@ -421,6 +435,42 @@ write_files:
421435
owner: "root"
422436
content: |
423437
CACERT
438+
- path: "/etc/kubernetes/ssl/etcd-apiserver-key.pem"
439+
permissions: "0644"
440+
encoding: "gzip+base64"
441+
owner: "root"
442+
content: |
443+
ETCDAPISERVERKEY
444+
- path: "/etc/kubernetes/ssl/etcd-apiserver.pem"
445+
permissions: "0664"
446+
encoding: "gzip+base64"
447+
owner: "root"
448+
content: |
449+
ETCDAPISERVER
450+
- path: "/etc/kubernetes/ssl/etcd-ca.pem"
451+
permissions: "0664"
452+
encoding: "gzip+base64"
453+
owner: "root"
454+
content: |
455+
ETCDCACERT
456+
- path: "/etc/ssl/certs/etcd-apiserver-key.pem"
457+
permissions: "0644"
458+
encoding: "gzip+base64"
459+
owner: "root"
460+
content: |
461+
ETCDAPISERVERKEY
462+
- path: "/etc/ssl/certs/etcd-apiserver.pem"
463+
permissions: "0664"
464+
encoding: "gzip+base64"
465+
owner: "root"
466+
content: |
467+
ETCDAPISERVER
468+
- path: "/etc/ssl/certs/etcd-ca.pem"
469+
permissions: "0664"
470+
encoding: "gzip+base64"
471+
owner: "root"
472+
content: |
473+
ETCDCACERT
424474
- path: /etc/motd.d/k8s.conf
425475
owner: "root"
426476
permissions: "0644"

‎template/worker.yaml‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,18 @@ coreos:
66
interface: WORKER_IP
77
etcd2:
88
discovery: https://discovery.etcd.io/DISCOVERY_ID
9-
advertise-client-urls: http://WORKER_IP:2379
10-
initial-advertise-peer-urls: http://WORKER_IP:2380
11-
listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
12-
listen-peer-urls: http://WORKER_IP:2380
9+
advertise-client-urls: https://WORKER_IP:2379
10+
initial-advertise-peer-urls: https://WORKER_IP:2380
11+
listen-client-urls: http://127.0.0.1:2379,https://WORKER_IP:2379
12+
listen-peer-urls: https://WORKER_IP:2380
13+
cert-file: /etc/kubernetes/ssl/etcd-worker.pem
14+
key-file: /etc/kubernetes/ssl/etcd-worker-key.pem
15+
trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
16+
client-cert-auth: true
17+
peer-cert-file: /etc/kubernetes/ssl/etcd-worker.pem
18+
peer-key-file: /etc/kubernetes/ssl/etcd-worker-key.pem
19+
peer-trusted-ca-file: /etc/kubernetes/ssl/etcd-ca.pem
20+
peer-client-cert-auth: true
1321
fleet:
1422
metadata: "role=node"
1523
units:
@@ -46,7 +54,7 @@ coreos:
4654
Requires=etcd2.service
4755
After=etcd2.service
4856
[Service]
49-
ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
57+
ExecStartPre=/usr/bin/etcdctl -cert-file=/etc/kubernetes/ssl/etcd-worker.pem --key-file=/etc/kubernetes/ssl/etcd-worker-key.pem --ca-file=/etc/kubernetes/ssl/etcd-ca.pem set /coreos.com/network/config '{ "Network": "10.2.0.0/16", "Backend":{"Type":"vxlan"}}'
5058
ExecStartPre=/usr/bin/ln -sf /etc/flannel/options.env /run/flannel/options.env
5159
command: start
5260
- name: kubelet.service
@@ -237,6 +245,9 @@ write_files:
237245
content: |
238246
FLANNELD_IFACE=WORKER_IP
239247
FLANNELD_ETCD_ENDPOINTS=ETCD_ENDPOINTS_URLS
248+
FLANNELD_ETCD_KEYFILE=/etc/ssl/certs/etcd-worker-key.pem
249+
FLANNELD_ETCD_CERTFILE=/etc/ssl/certs/etcd-worker.pem
250+
FLANNELD_ETCD_CAFILE=/etc/ssl/certs/etcd-ca.pem
240251
- path: "/etc/kubernetes/manifests/kube-proxy.yaml"
241252
permissions: "0644"
242253
owner: "root"
@@ -318,6 +329,42 @@ write_files:
318329
owner: "root"
319330
content: |
320331
WORKER
332+
- path: "/etc/kubernetes/ssl/etcd-ca.pem"
333+
permissions: "0664"
334+
encoding: "gzip+base64"
335+
owner: "root"
336+
content: |
337+
ETCDCACERT
338+
- path: "/etc/kubernetes/ssl/etcd-worker-key.pem"
339+
permissions: "0644"
340+
encoding: "gzip+base64"
341+
owner: "root"
342+
content: |
343+
ETCDWORKERKEY
344+
- path: "/etc/kubernetes/ssl/etcd-worker.pem"
345+
permissions: "0664"
346+
encoding: "gzip+base64"
347+
owner: "root"
348+
content: |
349+
ETCDWORKER
350+
- path: "/etc/ssl/certs/etcd-ca.pem"
351+
permissions: "0664"
352+
encoding: "gzip+base64"
353+
owner: "root"
354+
content: |
355+
ETCDCACERT
356+
- path: "/etc/ssl/certs/etcd-worker-key.pem"
357+
permissions: "0644"
358+
encoding: "gzip+base64"
359+
owner: "root"
360+
content: |
361+
ETCDWORKERKEY
362+
- path: "/etc/ssl/certs/etcd-worker.pem"
363+
permissions: "0664"
364+
encoding: "gzip+base64"
365+
owner: "root"
366+
content: |
367+
ETCDWORKER
321368
- path: /etc/motd.d/k8s.conf
322369
owner: "root"
323370
permissions: "0644"

0 commit comments

Comments
 (0)