Merge pull request #360 from cncf/feature/#322_node_failure

Add node failure resilience test cncf/cnf-conformance#322

Author: wvwatson

TEST-CATEGORIES.md

@@ -73,7 +73,7 @@ The CNF Conformance program enables interoperability of CNFs from multiple vendo
 
 ## Resilience Tests 
 [Cloud Native Definition](https://github.com/cncf/toc/blob/master/DEFINITION.md) requires systems to be Resilient to failures inevitable in cloud environments. CNF Resilience should be tested to ensure CNFs are designed to deal with non-carrier-grade shared cloud HW/SW platform:
-* For full failures in SW and HW platform: stopped cloud infrastructure/platform services, workload microservices or HW ingredients
+* For full failures in SW and HW platform: stopped cloud infrastructure/platform services, workload microservices or HW ingredients and nodes
 * For bursty, regular or partial impairments on key dependencies: CPU cycles by pausing, limiting or overloading; DPDK-based Dataplane networking by dropping and/or delaying packets.
 * Test if the CNF crashes when network loss occurs (Network Chaos)
 

USAGE.md

@@ -29,6 +29,11 @@ crystal build src/cnf-conformance.cr
 crystal src/cnf-conformance.cr all cnf-config=<path_to_your_config_file>/cnf-conformance.yml
 ```
 
+## Running all of the CNF Conformance tests (including proofs of concepts)
+``` 
+crystal src/cnf-conformance.cr all poc cnf-config=<path_to_your_config_file>/cnf-conformance.yml
+```
+
 ## Logging 
 
 ```
@@ -328,13 +333,23 @@ crystal src/cnf-conformance.cr chaos_container_kill
 ```
 
 ## Platform Tests
-#### (PoC) Run all platform tests
+#### Run all platform tests
 ```
 crystal src/cnf-conformance.cr platform
 ```
-#### (PoC) Run the K8s conformance tests
+#### Run the K8s conformance tests
 ```
 crystal src/cnf-conformance.cr k8s_conformance
 ```
+#### (PoC) Run All platform resilience tests 
+```
+crystal src/cnf-conformance.cr resilience poc
+
+```
+#### (PoC) Run node failure test **warning** this is a destructive test and will reboot your *host* node!
+#### Don't run this unless you have completely separate cluster (e.g. you are not running KIND on a dev box)
+```
+crystal src/cnf-conformance.cr node_failure poc destructive
+```
 
 

config.yml

@@ -8,5 +8,12 @@ toggles:
     toggle_on: false 
   - name: beta
     toggle_on: false
+  - name: poc 
+    toggle_on: false 
+# Don't change this to true unless you know what you are doing
+# I.E. your cluster and host (your dev box if you are using
+# kind) will be changed, rebooted, chaos tested, etc
+  - name: destructive 
+    toggle_on: false 
 loglevel: info
 

embedded_files/node_failure_values.yml

@@ -0,0 +1,10 @@
+tolerations:
+- key: "node.kubernetes.io/unreachable"
+  operator: "Exists"
+  effect: "NoExecute"
+  tolerationSeconds: 1
+- key: "node.kubernetes.io/not-ready"
+  operator: "Exists"
+  effect: "NoExecute"
+  tolerationSeconds: 1
+

points-all.yml

@@ -98,3 +98,7 @@
 
 - name: k8s_conformance
   tags: platform, dynamic
+- name: node_failure 
+  tags: platform, dynamic
+- name: recover_from_node_failure 
+  tags: platform, dynamic

points.yml

@@ -107,5 +107,9 @@
 #- name: performance
 #  tags: hardware, dynamic
 
-#- name: k8s_conformance
-#  tags: platform, dynamic
+# - name: k8s_conformance
+#   tags: platform, dynamic
+# - name: node_failure 
+#   tags: platform, dynamic
+# - name: recover_from_node_failure 
+#   tags: platform, dynamic

spec/platform/resilience_spec.cr

@@ -0,0 +1,29 @@
+require "./../spec_helper"
+require "colorize"
+require "./../../src/tasks/utils/utils.cr"
+
+describe "Platform" do
+  before_all do
+    # LOGGING.debug `pwd` 
+    # LOGGING.debug `echo $KUBECONFIG`
+    `./cnf-conformance samples_cleanup`
+    $?.success?.should be_true
+    `./cnf-conformance setup`
+    $?.success?.should be_true
+    `./cnf-conformance sample_coredns_with_wait_setup`
+    $?.success?.should be_true
+  end
+  it "'node_failure' should pass if chaos_mesh node_failure tests prove the platform is resilient" do
+    if check_destructive
+      puts "Tests running in destructive mode".colorize(:red) 
+      response_s = `./cnf-conformance platform:node_failure poc destructive`
+      LOGGING.info response_s
+      (/(PASSED: Node came back online)/ =~ response_s).should_not be_nil
+    else
+      response_s = `./cnf-conformance platform:node_failure poc`
+      LOGGING.info response_s
+      (/(PASSED: Nodes are resilient|Skipped)/ =~ response_s).should_not be_nil
+    end
+  end
+end
+

spec/resilience_spec.cr

@@ -7,8 +7,6 @@ require "sam"
 
 describe "Resilience" do
   before_all do
-    # `./cnf-conformance samples_cleanup force=true`
-    # $?.success?.should be_true
     `./cnf-conformance configuration_file_setup`
     $?.success?.should be_true
   end
@@ -18,6 +16,7 @@ describe "Resilience" do
       `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
       $?.success?.should be_true
       response_s = `./cnf-conformance chaos_container_kill verbose`
+      LOGGING.info response_s
       $?.success?.should be_true
       (/PASSED: Replicas available match desired count after container kill test/ =~ response_s).should_not be_nil
     ensure
@@ -31,6 +30,7 @@ describe "Resilience" do
       `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample-fragile-state deploy_with_chart=false`
       $?.success?.should be_true
       response_s = `./cnf-conformance chaos_container_kill verbose`
+      LOGGING.info response_s
       $?.success?.should be_true
       (/FAILURE: Replicas did not return desired count after container kill test/ =~ response_s).should_not be_nil
     ensure
@@ -44,6 +44,7 @@ describe "Resilience" do
       `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
       $?.success?.should be_true
       response_s = `./cnf-conformance chaos_network_loss verbose`
+      LOGGING.info response_s
       $?.success?.should be_true
       (/PASSED: Replicas available match desired count after network chaos test/ =~ response_s).should_not be_nil
     ensure
@@ -57,6 +58,7 @@ describe "Resilience" do
       `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false`
       $?.success?.should be_true
       response_s = `./cnf-conformance chaos_network_loss verbose`
+      LOGGING.info response_s
       $?.success?.should be_true
       (/FAILURE: Replicas did not return desired count after network chaos test/ =~ response_s).should_not be_nil
     ensure
@@ -70,6 +72,7 @@ describe "Resilience" do
       `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
       $?.success?.should be_true
       response_s = `./cnf-conformance chaos_cpu_hog verbose`
+      LOGGING.info response_s
       $?.success?.should be_true
       (/PASSED: Application pod is healthy after high CPU consumption/ =~ response_s).should_not be_nil
     ensure

src/tasks/chaos_mesh_setup.cr

@@ -0,0 +1,90 @@
+require "sam"
+require "file_utils"
+require "colorize"
+require "totem"
+require "./utils/utils.cr"
+
+CHAOS_MESH_VERSION = "v0.8.0"
+
+desc "Install Chaos Mesh"
+task "install_chaosmesh" do |_, args|
+  VERBOSE_LOGGING.info "install_chaosmesh" if check_verbose(args)
+  current_dir = FileUtils.pwd 
+  helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm"
+  crd_install = `kubectl create -f https://raw.githubusercontent.com/chaos-mesh/chaos-mesh/#{CHAOS_MESH_VERSION}/manifests/crd.yaml`
+  VERBOSE_LOGGING.info "#{crd_install}" if check_verbose(args)
+  unless Dir.exists?("#{current_dir}/#{TOOLS_DIR}/chaos_mesh")
+    # TODO use a tagged version
+    fetch_chaos_mesh = `git clone https://github.com/chaos-mesh/chaos-mesh.git #{current_dir}/#{TOOLS_DIR}/chaos_mesh`
+    checkout_tag = `cd #{current_dir}/#{TOOLS_DIR}/chaos_mesh && git checkout tags/#{CHAOS_MESH_VERSION} && cd -`
+  end
+  install_chaos_mesh = `#{helm} install chaos-mesh #{current_dir}/#{TOOLS_DIR}/chaos_mesh/helm/chaos-mesh --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock`
+  wait_for_resource("#{current_dir}/spec/fixtures/chaos_network_loss.yml")
+  wait_for_resource("#{current_dir}/spec/fixtures/chaos_cpu_hog.yml")
+  wait_for_resource("#{current_dir}/spec/fixtures/chaos_container_kill.yml")
+end
+
+desc "Uninstall Chaos Mesh"
+task "uninstall_chaosmesh" do |_, args|
+  VERBOSE_LOGGING.info "uninstall_chaosmesh" if check_verbose(args)
+  current_dir = FileUtils.pwd
+  helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm"
+  crd_delete = `kubectl delete -f https://raw.githubusercontent.com/chaos-mesh/chaos-mesh/#{CHAOS_MESH_VERSION}/manifests/crd.yaml`
+  FileUtils.rm_rf("#{current_dir}/#{TOOLS_DIR}/chaos_mesh")
+  delete_chaos_mesh = `#{helm} delete chaos-mesh`
+end
+
+def wait_for_test(test_type, test_name)
+  second_count = 0
+  wait_count = 60
+  status = ""
+  until (status.empty? != true && status == "Finished") || second_count > wait_count.to_i
+    LOGGING.debug "second_count = #{second_count}"
+    sleep 1
+    get_status = `kubectl get "#{test_type}" "#{test_name}" -o yaml`
+    LOGGING.info("#{get_status}")
+    status_data = Totem.from_yaml("#{get_status}")
+    LOGGING.info "Status: #{get_status}"
+    LOGGING.debug("#{status_data}")
+    status = status_data.get("status").as_h["experiment"].as_h["phase"].as_s
+    second_count = second_count + 1
+    LOGGING.info "#{get_status}"
+    LOGGING.info "#{second_count}"
+  end
+  # Did chaos mesh finish the test successfully
+  (status.empty? !=true && status == "Finished")
+end
+
+def desired_is_available?(deployment_name)
+  resp = `kubectl get deployments #{deployment_name} -o=yaml`
+  describe = Totem.from_yaml(resp)
+  LOGGING.info("desired_is_available describe: #{describe.inspect}")
+  desired_replicas = describe.get("status").as_h["replicas"].as_i
+  LOGGING.info("desired_is_available desired_replicas: #{desired_replicas}")
+  ready_replicas = describe.get("status").as_h["readyReplicas"]?
+  unless ready_replicas.nil?
+    ready_replicas = ready_replicas.as_i
+  else
+    ready_replicas = 0
+  end
+  LOGGING.info("desired_is_available ready_replicas: #{ready_replicas}")
+
+  desired_replicas == ready_replicas
+end
+
+def wait_for_resource(resource_file)
+  second_count = 0
+  wait_count = 60
+  is_resource_created = nil
+  until (is_resource_created.nil? != true && is_resource_created == true) || second_count > wait_count.to_i
+    LOGGING.info "second_count = #{second_count}"
+    sleep 3
+    `kubectl create -f #{resource_file} 2>&1 >/dev/null`
+    is_resource_created = $?.success?
+    LOGGING.info "Waiting for CRD"
+    LOGGING.info "Status: #{is_resource_created}"
+    LOGGING.debug "resource file: #{resource_file}"
+    second_count = second_count + 1
+  end
+  `kubectl delete -f #{resource_file}`
+end

src/tasks/platform/resilience.cr

@@ -0,0 +1,109 @@
+# coding: utf-8
+require "sam"
+require "colorize"
+require "../utils/utils.cr"
+
+namespace "platform" do
+  desc "The CNF conformance suite checks to see if the CNFs are resilient to failures."
+  task "resilience", ["node_failure"] do |t, args|
+    VERBOSE_LOGGING.info "resilience" if check_verbose(args)
+    VERBOSE_LOGGING.debug "resilience args.raw: #{args.raw}" if check_verbose(args)
+    VERBOSE_LOGGING.debug "resilience args.named: #{args.named}" if check_verbose(args)
+    stdout_score("resilience")
+  end
+
+  desc "Does the Platform recover the node and reschedule pods when a worker node fails"
+  task "node_failure" do |_, args|
+    unless check_poc(args) && check_destructive(args)
+      LOGGING.info "skipping node_failure: not in POC and destructive mode"
+      puts "Skipped".colorize(:yellow)
+      next
+    end
+    LOGGING.info "Running POC in destructive mode!"
+    task_response = task_runner(args) do |args|
+      current_dir = FileUtils.pwd 
+      helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm"
+
+      #Select the first node that isn't a master and is also schedulable
+      worker_nodes = `kubectl get nodes --selector='!node-role.kubernetes.io/master' -o 'go-template={{range .items}}{{$taints:=""}}{{range .spec.taints}}{{if eq .effect "NoSchedule"}}{{$taints = print $taints .key ","}}{{end}}{{end}}{{if not $taints}}{{.metadata.name}}{{ "\\n"}}{{end}}{{end}}'`
+      worker_node = worker_nodes.split("\n")[0]
+
+
+      File.write("node_failure_values.yml", NODE_FAILURE_VALUES)
+      install_coredns = `#{helm} install node-failure -f ./node_failure_values.yml --set nodeSelector."kubernetes\\.io/hostname"=#{worker_node} stable/coredns`
+      wait_for_install("node-failure-coredns")
+
+
+      File.write("reboot_daemon_pod.yml", REBOOT_DAEMON)
+      install_reboot_daemon = `kubectl create -f reboot_daemon_pod.yml`
+      wait_for_install("node-failure-coredns")
+
+      pod_ready = ""
+      pod_ready_timeout = 45
+      begin
+        until (pod_ready == "true" || pod_ready_timeout == 0)
+          pod_ready = pod_status("reboot", "--field-selector spec.nodeName=#{worker_node}").split(",")[2]
+          pod_ready_timeout = pod_ready_timeout - 1
+          if pod_ready_timeout == 0
+            upsert_failed_task("recover_from_node_failure", "✖️  FAILURE: Failed to install reboot daemon")
+            exit 1
+          end
+          sleep 1
+          puts "Waiting for reboot daemon to be ready"
+          puts "Reboot Daemon Ready Status: #{pod_ready}"
+        end
+
+        # Find Reboot Daemon name
+        reboot_daemon_pod = pod_status("reboot", "--field-selector spec.nodeName=#{worker_node}").split(",")[0]
+        start_reboot = `kubectl exec -ti #{reboot_daemon_pod} touch /tmp/reboot`
+
+        #Watch for Node Failure.
+        pod_ready = ""
+        node_ready = ""
+        node_failure_timeout = 30
+        until (pod_ready == "false" || node_ready == "False" || node_ready == "Unknown" || node_failure_timeout == 0)
+          pod_ready = pod_status("node-failure").split(",")[2]
+          node_ready = node_status("#{worker_node}")
+          puts "Waiting for Node to go offline"
+          puts "Pod Ready Status: #{pod_ready}"
+          puts "Node Ready Status: #{node_ready}"
+          node_failure_timeout = node_failure_timeout - 1
+          if node_failure_timeout == 0
+            upsert_failed_task("recover_from_node_failure", "✖️  FAILURE: Node failed to go offline")
+            exit 1
+          end
+          sleep 1
+        end
+
+        #Watch for Node to come back online
+        pod_ready = ""
+        node_ready = ""
+        node_online_timeout = 300
+        until (pod_ready == "true" && node_ready == "True" || node_online_timeout == 0)
+          pod_ready = pod_status("node-failure", "").split(",")[2]
+          node_ready = node_status("#{worker_node}")
+          puts "Waiting for Node to come back online"
+          puts "Pod Ready Status: #{pod_ready}"
+          puts "Node Ready Status: #{node_ready}"
+          node_online_timeout = node_online_timeout - 1
+          if node_online_timeout == 0
+            upsert_failed_task("recover_from_node_failure", "✖️  FAILURE: Node failed to come back online")
+            exit 1
+          end
+          sleep 1
+        end
+
+        emoji_chaos_network_loss="📶☠️"
+        resp = upsert_passed_task("recover_from_node_failure","✔️  PASSED: Node came back online #{emoji_chaos_network_loss}")
+
+
+      ensure
+        LOGGING.info "node_failure cleanup"
+        delete_reboot_daemon = `kubectl delete -f reboot_daemon_pod.yml`
+        delete_coredns = `#{helm} delete node-failure`
+        File.delete("reboot_daemon_pod.yml")
+        File.delete("node_failure_values.yml")
+      end
+    end
+  end
+end