Merge pull request #296 from dippynark/294-support-hard-and-soft-evic…

…tion

Support hard and soft evictions

puppet/modules/kubernetes/manifests/kubelet.pp

@@ -10,8 +10,21 @@
   String $role = 'worker',
   String $container_runtime = 'docker',
   String $kubelet_dir = '/var/lib/kubelet',
-  String $hard_eviction_memory_threshold =
-    five_percent_of_total_ram(dig44($facts, ['memory', 'system', 'total_bytes'], 1)),
+  Optional[String] $eviction_hard_memory_available_threshold = '5%',
+  Optional[String] $eviction_hard_nodefs_available_threshold = '10%',
+  Optional[String] $eviction_hard_nodefs_inodes_free_threshold = '5%',
+  Boolean $eviction_soft_enabled = true,
+  Optional[String] $eviction_soft_memory_available_threshold = '10%',
+  Optional[String] $eviction_soft_nodefs_available_threshold = '15%',
+  Optional[String] $eviction_soft_nodefs_inodes_free_threshold = '10%',
+  Optional[String] $eviction_soft_memory_available_grace_period = '0m',
+  Optional[String] $eviction_soft_nodefs_available_grace_period = '0m',
+  Optional[String] $eviction_soft_nodefs_inodes_free_grace_period = '0m',
+  String $eviction_max_pod_grace_period = '-1',
+  String $eviction_pressure_transition_period = '2m',
+  Optional[String] $eviction_minimum_reclaim_memory_available = '100Mi',
+  Optional[String] $eviction_minimum_reclaim_nodefs_available = '1Gi',
+  Optional[String] $eviction_minimum_reclaim_nodefs_inodes_free = undef,
   Optional[String] $network_plugin = undef,
   Integer $network_plugin_mtu = 1460,
   Boolean $allow_privileged = true,
@@ -43,6 +56,30 @@
 ){
   require ::kubernetes
 
+  if ! $eviction_soft_memory_available_threshold or ! $eviction_soft_memory_available_grace_period {
+    $_eviction_soft_memory_available_threshold = undef
+    $_eviction_soft_memory_available_grace_period = undef
+  } else {
+    $_eviction_soft_memory_available_threshold = $eviction_soft_memory_available_threshold
+    $_eviction_soft_memory_available_grace_period = $eviction_soft_memory_available_grace_period
+  }
+
+  if ! $eviction_soft_nodefs_available_threshold or ! $eviction_soft_nodefs_available_grace_period {
+    $_eviction_soft_nodefs_available_threshold = undef
+    $_eviction_soft_nodefs_available_grace_period = undef
+  } else {
+    $_eviction_soft_nodefs_available_threshold = $eviction_soft_nodefs_available_threshold
+    $_eviction_soft_nodefs_available_grace_period = $eviction_soft_nodefs_available_grace_period
+  }
+
+  if ! $eviction_soft_nodefs_inodes_free_threshold or ! $eviction_soft_nodefs_inodes_free_grace_period {
+    $_eviction_soft_nodefs_inodes_free_threshold = undef
+    $_eviction_soft_nodefs_inodes_free_grace_period = undef
+  } else {
+    $_eviction_soft_nodefs_inodes_free_threshold = $eviction_soft_nodefs_inodes_free_threshold
+    $_eviction_soft_nodefs_inodes_free_grace_period = $eviction_soft_nodefs_inodes_free_grace_period
+  }
+
   $_systemd_wants = $systemd_wants
   if $container_runtime == 'docker' {
     $_systemd_after = ['docker.service'] + $systemd_after

puppet/modules/kubernetes/spec/classes/kubelet_spec.rb

@@ -20,7 +20,43 @@
       should_not contain_file(service_file).with_content(/--network-plugin/)
       should contain_file(service_file).with_content(/--container-runtime=docker/)
       should contain_file(service_file).with_content(%r{--kubeconfig=/etc/kubernetes/kubeconfig-kubelet})
-      should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<191Mi})
+      should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<5%})
+      should contain_file(service_file).with_content(%r{--eviction-minimum-reclaim=memory.available=100Mi,nodefs.available=1Gi})
+      should contain_file(service_file).with_content(%r{--eviction-soft=memory.available<10%,nodefs.available<15%,nodefs.inodesFree<10%})
+      should contain_file(service_file).with_content(%r{--eviction-soft-grace-period=memory.available=0m,nodefs.available=0m,nodefs.inodesFree=0m})
+      should contain_file(service_file).with_content(%r{--eviction-max-pod-grace-period=-1})
+      should contain_file(service_file).with_content(%r{--eviction-pressure-transition-period=2m})
+    end
+  end
+
+  context 'without soft evictions' do
+    let(:params) { {
+      "eviction_soft_enabled" => false
+    }}
+    it do
+      should_not contain_file(service_file).with_content(%r{--eviction-soft})
+      should_not contain_file(service_file).with_content(%r{--eviction-soft-grace-period})
+      should_not contain_file(service_file).with_content(%r{--eviction-max-pod-grace-period})
+      should_not contain_file(service_file).with_content(%r{--eviction-pressure-transition-period})
+      should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<5%})
+      should contain_file(service_file).with_content(%r{--eviction-minimum-reclaim=memory.available=100Mi,nodefs.available=1Gi})
+    end
+  end
+
+  context 'soft evictions with modifications' do
+    let(:params) { {
+      "eviction_soft_memory_available_threshold" => '15%',
+      "eviction_minimum_reclaim_nodefs_available" => '2Gi',
+      "eviction_soft_nodefs_inodes_free_grace_period" => '1m',
+      "eviction_max_pod_grace_period" => '300',
+      "eviction_pressure_transition_period" => '5m'
+    }}
+    it do
+      should contain_file(service_file).with_content(%r{--eviction-soft=memory.available<15%,nodefs.available<15%,nodefs.inodesFree<10%})
+      should contain_file(service_file).with_content(%r{--eviction-minimum-reclaim=memory.available=100Mi,nodefs.available=2Gi})
+      should contain_file(service_file).with_content(%r{--eviction-soft-grace-period=memory.available=0m,nodefs.available=0m,nodefs.inodesFree=1m})
+      should contain_file(service_file).with_content(%r{--eviction-max-pod-grace-period=300})
+      should contain_file(service_file).with_content(%r{--eviction-pressure-transition-period=5m})
     end
   end
 

puppet/modules/kubernetes/templates/kubelet.service.erb

@@ -103,7 +103,53 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \
   "--tls-cert-file=<%= @cert_file %>" \
   "--tls-private-key-file=<%= @key_file %>" \
 <% end -%>
-  "--eviction-hard=memory.available<<%= @hard_eviction_memory_threshold %>" \
+<%
+    # build eviction hard command line
+    @eviction_hard = []
+    @eviction_hard << "memory.available<#{@eviction_hard_memory_available_threshold}" unless @eviction_hard_memory_available_threshold.nil? or @eviction_hard_memory_available_threshold == 'nil'
+    @eviction_hard << "nodefs.available<#{@eviction_hard_nodefs_available_threshold}" unless @eviction_hard_nodefs_available_threshold.nil? or @eviction_hard_nodefs_available_threshold == 'nil'
+    @eviction_hard << "nodefs.inodesFree<#{@eviction_hard_nodefs_inodes_free_threshold}" unless @eviction_hard_nodefs_inodes_free_threshold.nil? or @eviction_hard_nodefs_inodes_free_threshold == 'nil'
+-%>
+<% if @eviction_hard.length > 0 -%>
+  "--eviction-hard=<%= @eviction_hard.join(',') %>" \
+<% end -%>
+<% if @eviction_soft_enabled -%>
+<%
+    # build eviction soft command line
+    @eviction_soft = []
+
+    @eviction_soft << "memory.available<#{@_eviction_soft_memory_available_threshold}" unless @_eviction_soft_memory_available_threshold.nil? or @_eviction_soft_memory_available_threshold == 'nil'
+    @eviction_soft << "nodefs.available<#{@_eviction_soft_nodefs_available_threshold}" unless @_eviction_soft_nodefs_available_threshold.nil? or @_eviction_soft_nodefs_available_threshold == 'nil'
+    @eviction_soft << "nodefs.inodesFree<#{@_eviction_soft_nodefs_inodes_free_threshold}" unless @_eviction_soft_nodefs_inodes_free_threshold.nil? or @_eviction_soft_nodefs_inodes_free_threshold == 'nil'
+-%>
+<%    if @eviction_soft.length > 0 -%>
+<%
+    # build eviction soft grace period command line
+    @eviction_soft_grace_period = []
+
+    @eviction_soft_grace_period << "memory.available=#{@_eviction_soft_memory_available_grace_period}" unless @_eviction_soft_memory_available_grace_period.nil? or @_eviction_soft_memory_available_grace_period == 'nil'
+    @eviction_soft_grace_period << "nodefs.available=#{@_eviction_soft_nodefs_available_grace_period}" unless @_eviction_soft_nodefs_available_grace_period.nil? or @_eviction_soft_nodefs_available_grace_period == 'nil'
+    @eviction_soft_grace_period << "nodefs.inodesFree=#{@_eviction_soft_nodefs_inodes_free_grace_period}" unless @_eviction_soft_nodefs_inodes_free_grace_period.nil? or @_eviction_soft_nodefs_inodes_free_grace_period == 'nil'
+-%>
+<%        if @eviction_soft_grace_period.length > 0 -%>
+  --eviction-soft=<%= @eviction_soft.join(',') %> \
+  --eviction-soft-grace-period=<%= @eviction_soft_grace_period.join(',') %> \
+  --eviction-max-pod-grace-period=<%= @eviction_max_pod_grace_period %> \
+  --eviction-pressure-transition-period=<%= @eviction_pressure_transition_period %> \
+<%        end -%>
+<%    end -%>
+<% end -%>
+<%
+  # build minumum reclaim command line
+  @eviction_minimum_reclaim = []
+
+  @eviction_minimum_reclaim << "memory.available=#{@eviction_minimum_reclaim_memory_available}" unless @eviction_minimum_reclaim_memory_available.nil? or @eviction_minimum_reclaim_memory_available == 'nil'
+  @eviction_minimum_reclaim << "nodefs.available=#{@eviction_minimum_reclaim_nodefs_available}" unless @eviction_minimum_reclaim_nodefs_available.nil? or @eviction_minimum_reclaim_nodefs_available == 'nil'
+  @eviction_minimum_reclaim << "nodefs.inodesFree=#{@eviction_minimum_reclaim_nodefs_inodes_free}" unless @eviction_minimum_reclaim_nodefs_inodes_free.nil? or @eviction_minimum_reclaim_nodefs_inodes_free == 'nil'
+-%>
+<% if @eviction_minimum_reclaim.length > 0 -%>
+  "--eviction-minimum-reclaim=<%= @eviction_minimum_reclaim.join(',') %>" \
+<% end -%>
 <% if @_feature_gates != [] -%>
   --feature-gates=<%= @_feature_gates.join(',') %> \
 <% end -%>