diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..168da5f --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,2 @@ +exclude_paths: + - meta/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ccaf143 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,92 @@ +name: CI +'on': + pull_request: + push: + branches: + - master + +defaults: + run: + working-directory: 'rlex.k3s' + +jobs: + + yaml-lint: + name: YAML lint + runs-on: ubuntu-22.04 + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: 'rlex.k3s' + + - name: Set up Python 3 + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install test dependencies + run: pip3 install yamllint + + - name: Lint code. + run: | + yamllint . + + ansible-lint: + name: Ansible lint + runs-on: ubuntu-22.04 + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: 'rlex.k3s' + + - name: Set up Python 3 + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install test dependencies + run: pip3 install ansible-lint + + - name: Lint code + run: | + ansible-lint . + + molecule: + name: Molecule + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - distro: ubuntu2204 + scenario: single-node + - distro: debian11 + scenario: single-node + - distro: ubuntu2204 + scenario: cluster + - distro: debian11 + scenario: cluster + - distro: ubuntu2204 + scenario: extended-settings + + steps: + - name: Checkout codebase + uses: actions/checkout@v2 + with: + path: 'rlex.k3s' + + - name: Setup python 3 + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install test dependencies. + run: pip3 install ansible molecule molecule-plugins[docker] docker + + - name: Run Molecule tests. + run: molecule test --scenario-name "${{ matrix.scenario }}" + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + MOLECULE_DISTRO: ${{ matrix.distro }} diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000..9145122 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,9 @@ +--- + +yaml-files: + - '*.yaml' + - '*.yml' + - '.yamllint' + +rules: + line-length: disable diff --git a/defaults/main.yml b/defaults/main.yml index b45d2c4..f8f8a0a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -6,6 +6,7 @@ k3s_master_port: 6443 k3s_flannel_backend: vxlan k3s_master_disable: [] k3s_master_extra_args: [] +k3s_mount_bpffs: false k3s_agent_extra_args: "" k3s_agent_additional_config: "" k3s_master_additional_config: "" @@ -28,7 +29,7 @@ k3s_gvisor: false k3s_gvisor_platform: systrap k3s_gvisor_create_runtimeclass: true k3s_gvisor_config: {} -#https://github.com/google/gvisor/tags +# https://github.com/google/gvisor/tags k3s_gvisor_version: 20231113 k3s_sysctl_config: {} k3s_registries: "" @@ -36,7 +37,7 @@ k3s_kubeconfig: false k3s_kubeconfig_server: "{{ hostvars[groups[k3s_master_group][0]]['ansible_host'] | default(groups[k3s_master_group][0]) }}:{{ k3s_master_port }}" k3s_kubeconfig_context: k3s k3s_kubeconfig_target: "{{ k3s_kubeconfig_context }}" -#backwards compatibility +# backwards compatibility k3s_node_extra_args: "{{ k3s_agent_extra_args }}" k3s_node_external_ip: "{{ k3s_external_ip }}" k3s_node_ip: "{{ k3s_internal_ip }}" diff --git a/molecule/cluster/converge.yml b/molecule/cluster/converge.yml new file mode 100644 index 0000000..38c0882 --- /dev/null +++ b/molecule/cluster/converge.yml @@ -0,0 +1,20 @@ +--- +- name: Converge + hosts: all + vars: + k3s_internal_ip: "{{ ansible_default_ipv4.address }}" + k3s_agent_extra_args: + - "--snapshotter=native" + k3s_master_extra_args: + - "--snapshotter=native" + + pre_tasks: + - name: Update apt cache. + ansible.builtin.apt: + update_cache: true + cache_valid_time: 600 + when: ansible_os_family == 'Debian' + changed_when: false + + roles: + - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}" diff --git a/molecule/cluster/molecule.yml b/molecule/cluster/molecule.yml new file mode 100644 index 0000000..757296d --- /dev/null +++ b/molecule/cluster/molecule.yml @@ -0,0 +1,67 @@ +--- +role_name_check: 1 +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: master-1 + groups: + - k3s_master + image: "geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2204}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + pre_build_image: true + networks: + - name: k3s + - name: master-2 + groups: + - k3s_master + image: "geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2204}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + pre_build_image: true + networks: + - name: k3s + - name: master-3 + groups: + - k3s_master + image: "geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2204}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + pre_build_image: true + networks: + - name: k3s + - name: node-1 + groups: + - k3s_agent + image: "geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2204}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + pre_build_image: true + networks: + - name: k3s +provisioner: + name: ansible + options: + diff: true + inventory: + group_vars: + k3s_master: + k3s_master: true + k3s_agent: + k3s_agent: true + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} diff --git a/molecule/extended-settings/converge.yml b/molecule/extended-settings/converge.yml new file mode 100644 index 0000000..99d900a --- /dev/null +++ b/molecule/extended-settings/converge.yml @@ -0,0 +1,49 @@ +--- +- name: Converge + hosts: all + vars: + k3s_agent_extra_args: + - "--snapshotter=native" + k3s_master_extra_args: + - "--snapshotter=native" + # test adding additonal manifests + k3s_additional_manifests: + - name: kata + state: present + definition: + apiVersion: node.k8s.io/v1 + kind: RuntimeClass + metadata: + name: kata + handler: kata + # test adding additional configfiles + k3s_additional_config_files: + - name: apiserver-tracing.yaml + content: | + apiVersion: apiserver.config.k8s.io/v1alpha1 + kind: TracingConfiguration + endpoint: 127.0.0.1:4317 + samplingRatePerMillion: 100 + k3s_master_additional_config: + disable-cloud-controller: true + k3s_kubelet_additional_config: + - "cloud-provider=external" + k3s_additional_packages: + - open-iscsi + k3s_gvisor: true + k3s_gvisor_config: + network: host + k3s_server_disable: + - metrics-server + - traefik + + pre_tasks: + - name: Update apt cache. + ansible.builtin.apt: + update_cache: true + cache_valid_time: 600 + when: ansible_os_family == 'Debian' + changed_when: false + + roles: + - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}" diff --git a/molecule/extended-settings/molecule.yml b/molecule/extended-settings/molecule.yml new file mode 100644 index 0000000..a09f22c --- /dev/null +++ b/molecule/extended-settings/molecule.yml @@ -0,0 +1,29 @@ +--- +role_name_check: 1 +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: instance + groups: + - k3s_master + image: "geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2204}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + pre_build_image: true +provisioner: + name: ansible + options: + diff: true + inventory: + group_vars: + k3s_master: + k3s_master: true + k3s_agent: + k3s_agent: true + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} diff --git a/molecule/requirements.yml b/molecule/requirements.yml new file mode 100644 index 0000000..145989f --- /dev/null +++ b/molecule/requirements.yml @@ -0,0 +1 @@ +molecule-plugins[docker] diff --git a/molecule/single-node/converge.yml b/molecule/single-node/converge.yml new file mode 100644 index 0000000..bc82421 --- /dev/null +++ b/molecule/single-node/converge.yml @@ -0,0 +1,19 @@ +--- +- name: Converge + hosts: all + vars: + k3s_agent_extra_args: + - "--snapshotter=native" + k3s_master_extra_args: + - "--snapshotter=native" + + pre_tasks: + - name: Update apt cache. + ansible.builtin.apt: + update_cache: true + cache_valid_time: 600 + when: ansible_os_family == 'Debian' + changed_when: false + + roles: + - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}" diff --git a/molecule/single-node/molecule.yml b/molecule/single-node/molecule.yml new file mode 100644 index 0000000..a09f22c --- /dev/null +++ b/molecule/single-node/molecule.yml @@ -0,0 +1,29 @@ +--- +role_name_check: 1 +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: instance + groups: + - k3s_master + image: "geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2204}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + pre_build_image: true +provisioner: + name: ansible + options: + diff: true + inventory: + group_vars: + k3s_master: + k3s_master: true + k3s_agent: + k3s_agent: true + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} diff --git a/tasks/additional-configs.yml b/tasks/additional-configs.yml index c99a768..29f3c81 100644 --- a/tasks/additional-configs.yml +++ b/tasks/additional-configs.yml @@ -6,5 +6,5 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s with_items: "{{ k3s_additional_config_files }}" diff --git a/tasks/additional-masters.yml b/tasks/additional-masters.yml index 4aca0e2..92397fb 100644 --- a/tasks/additional-masters.yml +++ b/tasks/additional-masters.yml @@ -6,7 +6,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Copy k3s service file register: k3s_service @@ -16,7 +16,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Enable and start k3s (additional masters) ansible.builtin.systemd: diff --git a/tasks/agent.yml b/tasks/agent.yml index 0d771c2..a61e8d7 100644 --- a/tasks/agent.yml +++ b/tasks/agent.yml @@ -6,7 +6,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Copy k3s service file ansible.builtin.template: @@ -15,7 +15,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Enable and start k3s (workers) ansible.builtin.systemd: diff --git a/tasks/gvisor.yml b/tasks/gvisor.yml index 538302a..0550e25 100644 --- a/tasks/gvisor.yml +++ b/tasks/gvisor.yml @@ -13,7 +13,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Download gvisor runsc ansible.builtin.get_url: @@ -38,5 +38,5 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s # when: k3s_gvisor_config diff --git a/tasks/install.yml b/tasks/install.yml index 5fa3d57..fca8134 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -7,7 +7,7 @@ group: root mode: "0755" when: ansible_facts.architecture == "x86_64" - notify: restart k3s + notify: Restart k3s - name: Download k3s binary arm64 ansible.builtin.get_url: @@ -18,7 +18,7 @@ mode: "0755" when: - ( ansible_facts.architecture is search("arm") and ansible_facts.userspace_bits == "64" ) or ansible_facts.architecture is search("aarch64") - notify: restart k3s + notify: Restart k3s - name: Download k3s binary armhf ansible.builtin.get_url: @@ -30,4 +30,4 @@ when: - ansible_facts.architecture is search("arm") - ansible_facts.userspace_bits == "32" - notify: restart k3s + notify: Restart k3s diff --git a/tasks/kubeconfig.yml b/tasks/kubeconfig.yml index 7050f6c..4f93a49 100644 --- a/tasks/kubeconfig.yml +++ b/tasks/kubeconfig.yml @@ -1,4 +1,11 @@ --- +- name: Create directory .kube + ansible.builtin.file: + path: ~{{ ansible_user }}/.kube + state: directory + mode: "0700" + owner: "{{ ansible_user }}" + - name: Check if kubeconfig already exists ansible.builtin.stat: path: ~/.kube/config-{{ k3s_kubeconfig_target }} diff --git a/tasks/kuberesources.yml b/tasks/kuberesources.yml index 854f420..f61eff8 100644 --- a/tasks/kuberesources.yml +++ b/tasks/kuberesources.yml @@ -26,5 +26,6 @@ run_once: true delegate_to: "{{ hostvars[groups[k3s_master_group][0]]['inventory_hostname'] }}" when: - - k3s_additional_manifests - loop: "{{ k3s_additional_manifests }}" + - k3s_additional_manifests is iterable + - k3s_additional_manifests | length > 0 + loop: "{{ k3s_additional_manifests | list }}" diff --git a/tasks/main.yml b/tasks/main.yml index 73832ad..6067e06 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,27 +1,39 @@ --- -- ansible.builtin.import_tasks: preconf.yml -- ansible.builtin.import_tasks: additional-configs.yml -- ansible.builtin.import_tasks: gvisor.yml +- name: Run pre-configuration tasks + ansible.builtin.include_tasks: preconf.yml +- name: Create additional kubernetes configs + ansible.builtin.include_tasks: additional-configs.yml +- name: Run gvisor-related tasks + ansible.builtin.include_tasks: gvisor.yml when: - k3s_gvisor -- ansible.builtin.import_tasks: install.yml -- ansible.builtin.import_tasks: master.yml +- name: Install k3s distribution + ansible.builtin.include_tasks: install.yml +- name: Configure k3s master + ansible.builtin.include_tasks: master.yml when: - k3s_master - - k3s_master is defined - inventory_hostname == hostvars[groups[k3s_master_group][0]]['inventory_hostname'] -- ansible.builtin.import_tasks: additional-masters.yml +- name: Install additional master servers + ansible.builtin.include_tasks: additional-masters.yml when: - k3s_master - - k3s_master is defined - inventory_hostname != hostvars[groups[k3s_master_group][0]]['inventory_hostname'] -- ansible.builtin.import_tasks: agent.yml - when: k3s_agent and k3s_agent is defined -- ansible.builtin.import_tasks: utils.yml -- ansible.builtin.import_tasks: kubeconfig.yml +- name: Install k3s workers + ansible.builtin.include_tasks: agent.yml + when: + - k3s_agent +- name: Symlink k3s utils + ansible.builtin.include_tasks: utils.yml +- name: Download kubeconfig to local machine + ansible.builtin.include_tasks: kubeconfig.yml when: - k3s_kubeconfig - k3s_master - inventory_hostname == hostvars[groups[k3s_master_group][0]]['inventory_hostname'] -- ansible.builtin.import_tasks: cron.yml -- ansible.builtin.import_tasks: kuberesources.yml +- name: Create cronjobs + ansible.builtin.include_tasks: cron.yml + when: + - k3s_cronjob_prune_images == "present" +- name: Deploy additional kubernetes objects + ansible.builtin.include_tasks: kuberesources.yml diff --git a/tasks/master.yml b/tasks/master.yml index 5950bb1..3650a05 100644 --- a/tasks/master.yml +++ b/tasks/master.yml @@ -6,7 +6,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Copy k3s service file register: k3s_service @@ -16,7 +16,7 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s - name: Enable and start k3s (initial master) ansible.builtin.systemd: @@ -38,12 +38,6 @@ ansible.builtin.set_fact: k3s_node_token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" -- name: Create directory .kube - ansible.builtin.file: - path: ~{{ ansible_user }}/.kube - state: directory - owner: "{{ ansible_user }}" - - name: Create kubectl symlink ansible.builtin.file: src: /usr/local/bin/k3s diff --git a/tasks/preconf.yml b/tasks/preconf.yml index ab20bdf..205b00b 100644 --- a/tasks/preconf.yml +++ b/tasks/preconf.yml @@ -1,6 +1,7 @@ --- - name: Re-gather facts to get latest info ansible.builtin.setup: + - name: Enable IPv4 forwarding ansible.posix.sysctl: name: net.ipv4.ip_forward @@ -109,15 +110,14 @@ owner: root group: root mode: "0600" - notify: restart k3s + notify: Restart k3s when: k3s_registries - name: Delete k3s containerd config if gvisor is disabled ansible.builtin.file: - src: containerd.toml.tmpl.j2 path: /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl state: absent - notify: restart k3s + notify: Restart k3s when: - not k3s_gvisor diff --git a/templates/k3s-config.yaml.j2 b/templates/k3s-config.yaml.j2 index 97bec43..4b6f23c 100644 --- a/templates/k3s-config.yaml.j2 +++ b/templates/k3s-config.yaml.j2 @@ -3,7 +3,7 @@ {% if inventory_hostname == hostvars[groups[k3s_master_group][0]]['inventory_hostname'] %} cluster-init: true {% endif %} -{% if k3s_master_disable is defined and k3s_master_disable is iterable %} +{% if k3s_master_disable is iterable and k3s_master_disable | length > 0 %} disable: {{ k3s_master_disable | to_nice_yaml(indent=2) }} {% endif %}