Merge pull request #1835 from cncf/bug/1834

[bug/1834] Uncordon node after running `node_drain` test for a CNF

Author: agentpoyo

sample-cnfs/sample-coredns-cnf2/README.md

@@ -0,0 +1,39 @@
+# Set up Sample CoreDNS CNF
+./sample-cnfs/sample-coredns-cnf/readme.md
+# Prerequistes
+### Install helm
+```
+curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+chmod 700 get_helm.sh
+./get_helm.sh
+```
+### Optional: Use a helm version manager
+https://github.com/yuya-takeyama/helmenv
+Check out helmenv into any path (here is ${HOME}/.helmenv)
+```
+${HOME}/.helmenv)
+$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv
+```
+Add ~/.helmenv/bin to your $PATH any way you like
+```
+$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile
+```
+```
+helmenv versions 
+helmenv install <version 3.1?>
+```
+
+### core-dns installation
+```
+helm install coredns stable/coredns
+```
+### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory
+```
+helm pull stable/coredns
+```
+### Example cnf-testsuite config file for sample-core-dns-cnf
+In ./cnfs/sample-core-dns-cnf/cnf-testsuite.yml
+```
+---
+container_names: [coredns-coredns] 
+```

sample-cnfs/sample-coredns-cnf2/cnf-testsuite.yml

@@ -0,0 +1,9 @@
+---
+release_name: coredns2
+service_name: coredns-coredns 
+helm_repository:
+  name: stable 
+  repo_url: https://cncf.gitlab.io/stable
+helm_chart: stable/coredns
+helm_install_namespace: cnfspace2
+allowlist_helm_chart_container_names: [falco, node-cache, nginx, coredns, calico-node, kube-proxy, nginx-proxy]

shard.lock

@@ -54,7 +54,7 @@ shards:
 
   kubectl_client:
     git: https://github.com/cnf-testsuite/kubectl_client.git
-    version: 1.0.3
+    version: 1.0.4
 
   popcorn:
     git: https://github.com/icyleaf/popcorn.git

shard.yml

@@ -49,7 +49,7 @@ dependencies:
     version: ~> 1.0.0
   kubectl_client:
     github: cnf-testsuite/kubectl_client
-    version: ~> 1.0.3
+    version: ~> 1.0.4
   cluster_tools:
     github: cnf-testsuite/cluster_tools
     version: ~> 1.0.0

src/tasks/litmus_setup.cr

@@ -76,15 +76,12 @@ module LitmusManager
     File.write(MODIFIED_LITMUS_FILE, output_file) unless output_file == nil
   end
 
-  def self.cordon_target_node(deployment_label, deployment_value, namespace)
+  def self.get_target_node_to_cordon(deployment_label, deployment_value, namespace)
     app_nodeName_cmd = "kubectl get pods -l #{deployment_label}=#{deployment_value} -n #{namespace} -o=jsonpath='{.items[0].spec.nodeName}'"
     Log.info { "Getting the operator node name: #{app_nodeName_cmd}" }
     status_code = Process.run("#{app_nodeName_cmd}", shell: true, output: appNodeName_response = IO::Memory.new, error: stderr = IO::Memory.new).exit_status
     Log.for("verbose").info { "status_code: #{status_code}" } 
-    app_nodeName = appNodeName_response.to_s 
-    status_code = KubectlClient::Cordon.command("#{app_nodeName}")
-    Log.for("verbose").info { "status_code: #{status_code}" }
-    Log.info { "The target node has been cordoned sucessfully" }
+    appNodeName_response.to_s
   end
 
   ## wait_for_test will wait for the completion of litmus test

src/tasks/workload/state.cr

@@ -229,10 +229,29 @@ task "node_drain", ["install_litmus"] do |t, args|
       spec_labels = KubectlClient::Get.resource_spec_labels(resource["kind"], resource["name"], resource["namespace"])
 
       schedulable_nodes_count=KubectlClient::Get.schedulable_nodes_list
+      cordon_deployment_label = "#{spec_labels.as_h.first_key}"
+      cordon_deployment_value = "#{spec_labels.as_h.first_value}"
+
+      # Declare this outside the block so that the name of the node can be used to uncordon later.
+      cordon_target_node_name = nil
+
       if schedulable_nodes_count.size > 1
-        LitmusManager.cordon_target_node("#{spec_labels.as_h.first_key}","#{spec_labels.as_h.first_value}", namespace: resource["namespace"])
+        # Identify cordon node target.
+        cordon_target_node_name = LitmusManager.get_target_node_to_cordon(cordon_deployment_label, cordon_deployment_value, namespace: app_namespace)
+        Log.info { "Found node to cordon #{cordon_target_node_name} using label #{cordon_deployment_label}='#{cordon_deployment_value}' in #{app_namespace} namespace." }
+
+        # Cordon the node.
+        result = KubectlClient::Cordon.command("#{cordon_target_node_name}")
+
+        # If cordoning fails, skip the test.
+        if result[:status].success?
+          Log.info { "Cordoned node #{cordon_target_node_name} successfully." }
+        else
+          Log.info { "Unable to cordon node #{cordon_target_node_name}." }
+          skipped = true
+        end
       else
-        Log.info { "The target node was unable to cordoned sucessfully" }
+        Log.info { "Skipping test. Scheduleable node count is not > 1." }
         skipped = true
       end
 
@@ -328,7 +347,19 @@ task "node_drain", ["install_litmus"] do |t, args|
           LitmusManager.wait_for_test(test_name,chaos_experiment_name,total_chaos_duration,args, namespace: app_namespace)
           test_passed = LitmusManager.check_chaos_verdict(chaos_result_name,chaos_experiment_name,args, namespace: app_namespace)
         end
+
+        # Uncordon the node.
+        result = KubectlClient::Uncordon.command("#{cordon_target_node_name}")
+
+        # If uncordoning fails, log the error.
+        if result[:status].success?
+          Log.info { "Uncordoned node #{cordon_target_node_name} successfully." }
+        else
+          Log.error { "Uncordoning node #{cordon_target_node_name} failed." }
+          skipped = true
+        end
       end
+
       test_passed
     end
     if skipped