Skip to content

Commit baa8f99

Browse files
lbricklbrick
committed
bring in the single k8s master stuff
1 parent f2ebefe commit baa8f99

File tree

25 files changed

+628
-24
lines changed

25 files changed

+628
-24
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ terraform/.terraform.lock.hcl
22
terraform/.terraform/*
33
host.ini
44
terraform/terraform.tfstate
5+
terraform/terraform.tfstate.d/*
56
terraform/terraform.tfstate.backup
67
terraform/terraform.tfvars
78
env.sh

‎config/.gitkeep

Whitespace-only changes.

‎group_vars/k8s_cluster/k8s_cluster.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# K8s cluster name
2+
cluster_name: "training-k8s"
3+
4+
# Change this to use another Kubernetes version, e.g. a current beta release
5+
apt_kube_version: 1.23.0-00
6+
kube_version: v1.23.4
7+
8+
kube_apiserver_bind_address: 0.0.0.0
9+
kube_apiserver_port: 6443 # (https)
10+
11+
# Kubernetes internal network for services, unused block of space.
12+
kube_service_addresses: 10.233.0.0/18
13+
14+
# internal network. When used, it will assign IP
15+
# addresses from this range to individual pods.
16+
# This network must be unused in your network infrastructure!
17+
kube_pods_subnet: 10.233.64.0/18
18+
19+
# A port range to reserve for services with NodePort visibility.
20+
# Inclusive at both ends of the range.
21+
kube_apiserver_node_port_range: "30000-32767"
22+
23+
# Kubernetes settings
24+
kube_api_anonymous_auth: true
25+
26+
## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
27+
## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
28+
29+
kube_oidc_auth: true
30+
kube_oidc_url: https://ood-idp.training.data.nesi.org.nz/realms/ondemand
31+
kube_oidc_client_id: kubernetes
32+
## Optional settings for OIDC
33+
kube_oidc_username_claim: preferred_username
34+
kube_oidc_username_prefix: "-"
35+
kube_oidc_groups_claim: groups
36+
kube_oidc_groups_prefix: 'oidc:'
37+
# Copy oidc CA file to the following path if needed
38+
kube_oidc_ca_file: "{{ kube_cert_dir }}/zero_ca.crt"
39+
# Optionally include a base64-encoded oidc CA cert
40+
kube_oidc_ca_cert: <I_WILL_PASS_YOU_THIS-MAYBE_SET_AS_AN_ENV>

‎roles/docker/tasks/main.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
- name: Creates directory
3+
file:
4+
path: /etc/docker
5+
state: directory
6+
7+
- name: Copy daemon.json to nodes
8+
copy:
9+
src: ../template/daemon.json
10+
dest: /etc/docker/daemon.json
11+
12+
- name: Docker Installation
13+
apt:
14+
name: docker.io
15+
state: present
16+
update_cache: true

‎roles/docker/template/daemon.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"exec-opts": ["native.cgroupdriver=systemd"],
3+
"log-driver": "json-file",
4+
"log-opts": {
5+
"max-size": "100m"
6+
},
7+
"storage-driver": "overlay2"
8+
}

‎roles/infra/wait-for-hosts/tasks/main.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
- name: Wait for nodes to become accessible
3+
wait_for_connection:
4+
# Wait for 10 mins for host to become available
5+
timeout: 30
6+
vars:
7+
ansible_connection: ssh
8+
ansible_host: "{{ hostvars[item]['ansible_host'] }}"
9+
ansible_user: "ubuntu"
10+
ansible_ssh_private_key_file: "~/.ssh/id_flexi"
11+
ansible_python_interpreter: auto_legacy
12+
ansible_ssh_common_args: "{{ hostvars[item]['ansible_ssh_common_args'] }}"
13+
with_items:
14+
- "{{ groups['all'] }}"

‎roles/kubernetes/control-plane/defaults/main.yml

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
cluster_name: "kubernetes"
2+
3+
## Change this to use another Kubernetes version, e.g. a current beta release
4+
apt_kube_version: 1.23.0-00
5+
kube_version: v1.23.4
6+
7+
kube_apiserver_bind_address: 0.0.0.0
8+
kube_apiserver_port: 6443 # (https)
9+
10+
# Kubernetes internal network for services, unused block of space.
11+
kube_service_addresses: 10.233.0.0/18
12+
13+
# internal network. When used, it will assign IP
14+
# addresses from this range to individual pods.
15+
# This network must be unused in your network infrastructure!
16+
kube_pods_subnet: 10.233.64.0/18
17+
18+
# A port range to reserve for services with NodePort visibility.
19+
# Inclusive at both ends of the range.
20+
kube_apiserver_node_port_range: "30000-32767"
21+
22+
is_kube_master: "{{ inventory_hostname in groups['kube_control_plane'] }}"
23+
kube_apiserver_count: "{{ groups['kube_control_plane'] | length }}"
24+
kube_apiserver_address: "{{ ansible_host | default(fallback_ips[inventory_hostname]) }}"
25+
first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['ansible_host'] }}"
26+
kube_apiserver_endpoint: https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
27+
28+
# Kubernetes settings
29+
kube_api_anonymous_auth: true
30+
31+
# Editing those values will almost surely break something.
32+
kube_config_dir: /etc/kubernetes
33+
kube_manifest_dir: "{{ kube_config_dir }}/manifests"
34+
35+
# This is where all the cert scripts and certs will be located
36+
kube_cert_dir: "{{ kube_config_dir }}/ssl"
37+
38+
## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
39+
## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
40+
41+
kube_oidc_auth: false
42+
kube_oidc_url: https://ood-idp.training.data.nesi.org.nz/realms/ondemand
43+
kube_oidc_client_id: kubernetes
44+
## Optional settings for OIDC
45+
kube_oidc_username_claim: preferred_username
46+
kube_oidc_username_prefix: "-"
47+
kube_oidc_groups_claim: groups
48+
kube_oidc_groups_prefix: 'oidc:'
49+
# Copy oidc CA file to the following path if needed
50+
kube_oidc_ca_file: "{{ kube_cert_dir }}/zero_ca.crt"
51+
# Optionally include a base64-encoded oidc CA cert
52+
kube_oidc_ca_cert: LS0tLS1CRU.....

‎roles/kubernetes/control-plane/tasks/control-plane-setup.yml

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
---
2+
- name: Create Kubernetes ssl directory
3+
file:
4+
path: "{{ kube_cert_dir }}"
5+
state: directory
6+
7+
- name: Install OIDC certificate
8+
copy:
9+
content: "{{ kube_oidc_ca_cert | b64decode }}"
10+
dest: "{{ kube_oidc_ca_file }}"
11+
owner: root
12+
group: root
13+
mode: "0644"
14+
when:
15+
- kube_oidc_auth
16+
- kube_oidc_ca_cert is defined
17+
18+
- name: Check if kubeadm has already run
19+
stat:
20+
path: "/var/lib/kubelet/config.yaml"
21+
get_attributes: no
22+
get_checksum: no
23+
get_mime: no
24+
register: kubeadm_already_run
25+
26+
- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
27+
shell: >-
28+
kubeadm token generate
29+
changed_when: false
30+
when:
31+
- inventory_hostname == first_kube_control_plane
32+
- kubeadm_token is defined
33+
register: temp_token
34+
35+
- name: Set kubeadm_token
36+
set_fact:
37+
kubeadm_token: "{{ temp_token.stdout_lines[0] }}"
38+
when:
39+
- temp_token.stdout is defined
40+
- inventory_hostname == first_kube_control_plane
41+
42+
- name: Create kubeadm config
43+
template:
44+
src: "kubeadm-config.yaml.j2"
45+
dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
46+
mode: 0640
47+
when:
48+
- inventory_hostname == first_kube_control_plane
49+
50+
- name: Initialize first master
51+
command: >-
52+
kubeadm init
53+
--config={{ kube_config_dir }}/kubeadm-config.yaml
54+
--ignore-preflight-errors=all
55+
--upload-certs
56+
when:
57+
- inventory_hostname == first_kube_control_plane
58+
- not kubeadm_already_run.stat.exists
59+
60+
- name: create .kube directory
61+
file:
62+
path: $HOME/.kube
63+
state: directory
64+
mode: 0755
65+
when:
66+
- inventory_hostname == first_kube_control_plane
67+
- not kubeadm_already_run.stat.exists
68+
69+
- name: copy admin.conf to user's kube config
70+
copy:
71+
src: /etc/kubernetes/admin.conf
72+
dest: $HOME/.kube/config
73+
remote_src: yes
74+
when:
75+
- inventory_hostname == first_kube_control_plane
76+
- not kubeadm_already_run.stat.exists
77+
78+
- name: copy admin.conf to loacal
79+
fetch:
80+
src: /etc/kubernetes/admin.conf
81+
dest: ./config/k8s-admin.conf
82+
flat: yes
83+
when:
84+
- inventory_hostname == first_kube_control_plane
85+
- not kubeadm_already_run.stat.exists
86+
87+
- name: install Pod network
88+
command: >-
89+
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
90+
when:
91+
- inventory_hostname == first_kube_control_plane
92+
- not kubeadm_already_run.stat.exists
93+
94+
# - name: Create a New Certificate Key
95+
# become: true
96+
# command: >-
97+
# kubeadm init phase upload-certs --upload-certs
98+
# register: join_certificate_key
99+
# changed_when: join_certificate_key.rc != 0 # Uses the return code to define when the task has changed.
100+
# when:
101+
# - inventory_hostname == first_kube_control_plane
102+
103+
# - name: Generate Master Join Command
104+
# become: true
105+
# command: >-
106+
# kubeadm token create --print-join-command --certificate-key "{{ join_certificate_key.stdout_lines[2] }}"
107+
# register: join_master_command_raw
108+
# changed_when: join_master_command.rc != 0 # Uses the return code to define when the task has changed.
109+
# when:
110+
# - inventory_hostname == first_kube_control_plane
111+
112+
# - name: Set Master Join Command
113+
# set_fact:
114+
# master_join_command: "{{ join_master_command_raw.stdout_lines[0] }}"
115+
# when:
116+
# - inventory_hostname == first_kube_control_plane

‎roles/kubernetes/control-plane/tasks/define-first-kube-control.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
- name: Check which kube-control nodes are already members of the cluster
3+
command: "kubectl get nodes --selector=node-role.kubernetes.io/control-plane -o json"
4+
register: kube_control_planes_raw
5+
ignore_errors: true
6+
changed_when: false
7+
8+
- name: Set fact joined_control_panes
9+
set_fact:
10+
joined_control_planes: "{{ ((kube_control_planes_raw.stdout | from_json)['items']) | default([]) | map(attribute='metadata') | map(attribute='name') | list }}"
11+
delegate_to: item
12+
loop: "{{ groups['kube_control_plane'] }}"
13+
when: kube_control_planes_raw is succeeded
14+
run_once: true
15+
16+
- name: Set fact first_kube_control_plane
17+
set_fact:
18+
first_kube_control_plane: "{{ joined_control_planes | default([]) | first | default(groups['kube_control_plane'] | first) }}"

‎roles/kubernetes/control-plane/tasks/main.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
- name: Install Kubectl
3+
apt:
4+
name: kubectl={{ apt_kube_version }}
5+
state: present
6+
force: yes
7+
8+
- name: Define nodes already joined to existing cluster and first_kube_control_plane
9+
import_tasks: define-first-kube-control.yml
10+
11+
- name: Setup first master
12+
include_tasks: control-plane-setup.yml

0 commit comments

Comments
 (0)