feat!: replace calico with cilium (#740)

README.md

@@ -31,7 +31,7 @@ The following components will be installed in your [k3s](https://k3s.io/) cluste
 - [kube-vip](https://kube-vip.io/) - Load balancer for the Kubernetes control plane nodes
 - [metallb](https://metallb.universe.tf/) - Load balancer for Kubernetes services
 - [cert-manager](https://cert-manager.io/) - Operator to request SSL certificates and store them as Kubernetes resources
-- [calico](https://www.tigera.io/project-calico/) - Container networking interface for inter pod and service networking
+- [cilium](https://cilium.io/) - Container networking interface for inter pod and service networking
 - [external-dns](https://github.com/kubernetes-sigs/external-dns) - Operator to publish DNS records to Cloudflare (and other providers) based on Kubernetes ingresses
 - [k8s_gateway](https://github.com/ori-edge/k8s_gateway) - DNS resolver that provides local DNS to your Kubernetes ingresses
 - [ingress-nginx](https://kubernetes.github.io/ingress-nginx/) - Kubernetes ingress controller used for a HTTP reverse proxy of Kubernetes ingresses

ansible/inventory/group_vars/kubernetes/k3s.yml

@@ -31,9 +31,6 @@ k3s_server_manifests_urls:
  # Kube-vip
  - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml
  filename: custom-kube-vip-rbac.yaml
- # Tigera Operator
- - url: https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml
- filename: custom-calico-tigera-operator.yaml
  # Essential Prometheus Operator CRDs
  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
  filename: custom-prometheus-podmonitors.yaml
@@ -42,8 +39,12 @@ k3s_server_manifests_urls:
  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
  filename: custom-prometheus-servicemonitors.yaml
 
-# (list) A flat list of templates to deploy on the primary control plane
+# (list) A flat list of templates to deploy on the primary control plane node
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
- - custom-calico-installation.yaml.j2
- - custom-kube-vip-daemonset.yaml.j2
+ - cilium-helmchart.yaml.j2
+
+# (list) A flat list of templates to deploy as static pods on all the control plane nodes
+# /var/lib/rancher/k3s/agent/pod-manifests
+k3s_server_pod_manifests_templates:
+ - kube-vip-static-pod.yaml.j2

ansible/inventory/group_vars/master/k3s.yml

@@ -26,6 +26,7 @@ k3s_server:
  - traefik
  disable-network-policy: true
  disable-cloud-controller: true
+ disable-kube-proxy: true
  write-kubeconfig-mode: "644"
  # Network CIDR to use for pod IPs
  cluster-cidr: "10.42.0.0/16"
@@ -34,9 +35,6 @@ k3s_server:
  kube-controller-manager-arg:
  # Required to monitor kube-controller-manager with kube-prometheus-stack
  - "bind-address=0.0.0.0"
- kube-proxy-arg:
- # Required to monitor kube-proxy with kube-prometheus-stack
- - "metrics-bind-address=0.0.0.0"
  kube-scheduler-arg:
  # Required to monitor kube-scheduler with kube-prometheus-stack
  - "bind-address=0.0.0.0"

ansible/playbooks/cluster-installation.yml

@@ -68,14 +68,9 @@
  wait_sleep: 10
  wait_timeout: 360
  loop:
- - name: tigera-operator
- kind: Deployment
- namespace: tigera-operator
- - name: kube-vip
- kind: DaemonSet
+ - name: cilium
+ kind: HelmChart
  namespace: kube-system
- - kind: Installation
- name: default
  - name: podmonitors.monitoring.coreos.com
  kind: CustomResourceDefinition
  - name: prometheusrules.monitoring.coreos.com
@@ -89,7 +84,7 @@
  - k3s_control_node
 
  # NOTE
- # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests directory
+ # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory
  # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s
  # service is restarted. Removing them does not uninstall the manifests from your cluster.
 

ansible/playbooks/templates/cilium-helmchart.yaml.j2

@@ -0,0 +1,54 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: cilium
+ namespace: kube-system
+spec:
+ repo: https://helm.cilium.io/
+ chart: cilium
+ version: 1.13.2
+ targetNamespace: kube-system
+ bootstrap: true
+ valuesContent: |-
+ bpf:
+ masquerade: true
+ cluster:
+ name: home-cluster
+ id: 1
+ containerRuntime:
+ integration: containerd
+ socketPath: /var/run/k3s/containerd/containerd.sock
+ hubble:
+ enabled: true
+ metrics:
+ enabled:
+ - dns:query;ignoreAAAA
+ - drop
+ - tcp
+ - flow
+ - port-distribution
+ - icmp
+ - http
+ relay:
+ enabled: true
+ rollOutPods: true
+ ui:
+ enabled: true
+ rollOutPods: true
+ ingress:
+ enabled: false
+ ipam:
+ mode: cluster-pool
+ operator:
+ clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"]
+ clusterPoolIPv4MaskSize: 24
+ k8sServiceHost: "{{ kubevip_address }}"
+ k8sServicePort: 6443
+ kubeProxyReplacement: strict
+ kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+ operator:
+ enabled: true
+ replicas: 1
+ rollOutPods: true
+ rollOutCiliumPods: true

ansible/playbooks/templates/kube-vip-static-pod.yaml.j2

@@ -0,0 +1,58 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: kube-vip
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/instance: kube-vip
+ app.kubernetes.io/name: kube-vip
+spec:
+ containers:
+ - name: kube-vip
+ image: ghcr.io/kube-vip/kube-vip:v0.6.0
+ imagePullPolicy: IfNotPresent
+ args: ["manager"]
+ env:
+ - name: address
+ value: "{{ kubevip_address }}"
+ - name: vip_arp
+ value: "true"
+ - name: port
+ value: "6443"
+ - name: vip_cidr
+ value: "32"
+ - name: cp_enable
+ value: "true"
+ - name: cp_namespace
+ value: kube-system
+ - name: vip_ddns
+ value: "false"
+ - name: svc_enable
+ value: "false"
+ - name: vip_leaderelection
+ value: "true"
+ - name: vip_leaseduration
+ value: "15"
+ - name: vip_renewdeadline
+ value: "10"
+ - name: vip_retryperiod
+ value: "2"
+ - name: prometheus_server
+ value: :2112
+ securityContext:
+ capabilities:
+ add: ["NET_ADMIN", "NET_RAW"]
+ volumeMounts:
+ - mountPath: /etc/kubernetes/admin.conf
+ name: kubeconfig
+ hostAliases:
+ - hostnames:
+ - kubernetes
+ ip: 127.0.0.1
+ hostNetwork: true
+ serviceAccountName: kube-vip
+ volumes:
+ - name: kubeconfig
+ hostPath:
+ path: /etc/rancher/k3s/k3s.yaml