Merge pull request #376 from cncf/feature/#327_platform_totals

#327 platform totals

Author: denverwilliams

config.yml

@@ -14,6 +14,6 @@ toggles:
 # I.E. your cluster and host (your dev box if you are using
 # kind) will be changed, rebooted, chaos tested, etc
   - name: destructive 
-    toggle_on: false 
+    toggle_on: false
 loglevel: info
 

points-all.yml

@@ -101,6 +101,6 @@
 - name: node_failure 
   tags: platform, dynamic
 - name: recover_from_node_failure 
-  tags: platform, dynamic
+  tags: platform, platform:resilience, dynamic
 - name: oci_compliant
-  tags: platform, dynamic
+  tags: platform, platform:hardware_and_scheduling, dynamic

points.yml

@@ -107,11 +107,9 @@
 #- name: performance
 #  tags: hardware, dynamic
 
-# - name: k8s_conformance
-#   tags: platform, dynamic
-# - name: node_failure 
-#   tags: platform, dynamic
-# - name: recover_from_node_failure 
-#   tags: platform, dynamic
-- name: oci_compliant
+- name: k8s_conformance
   tags: platform, dynamic
+- name: recover_from_node_failure 
+  tags: platform, platform:resilience, dynamic
+- name: oci_compliant
+  tags: platform, platform:hardware_and_scheduling, dynamic

spec/utils/utils_spec.cr

@@ -112,7 +112,7 @@ describe "Utils" do
 
   it "'all_task_test_names' should return all tasks names" do
     clean_results_yml
-    (all_task_test_names()).should eq(["reasonable_image_size", "reasonable_startup_time", "privileged", "increase_capacity", "decrease_capacity", "network_chaos", "ip_addresses", "liveness", "readiness", "rolling_update", "nodeport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "helm_deploy", "install_script_helm", "helm_chart_valid", "helm_chart_published", "chaos_network_loss", "chaos_cpu_hog", "chaos_container_kill", "volume_hostpath_not_found", "oci_compliant"])
+    (all_task_test_names()).should eq(["reasonable_image_size", "reasonable_startup_time", "privileged", "increase_capacity", "decrease_capacity", "network_chaos", "ip_addresses", "liveness", "readiness", "rolling_update", "nodeport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "helm_deploy", "install_script_helm", "helm_chart_valid", "helm_chart_published", "chaos_network_loss", "chaos_cpu_hog", "chaos_container_kill", "volume_hostpath_not_found"])
   end
 
   it "'all_result_test_names' should return the tasks assigned to a tag" do

src/tasks/platform/hardware_and_scheduling.cr

@@ -9,7 +9,7 @@ namespace "platform" do
     VERBOSE_LOGGING.info "hardware_and_scheduling" if check_verbose(args)
     VERBOSE_LOGGING.debug "hardware_and_scheduling args.raw: #{args.raw}" if check_verbose(args)
     VERBOSE_LOGGING.debug "hardware_and_scheduling args.named: #{args.named}" if check_verbose(args)
-    stdout_score("resilience")
+    stdout_score("platform:hardware_and_scheduling")
   end
 
   desc "Does the Platform use a runtime that is oci compliant"

src/tasks/platform/platform.cr

@@ -1,6 +1,7 @@
 desc "Platform Tests"
-task "platform", ["k8s_conformance"]  do |_, args|
+task "platform", ["k8s_conformance", "platform:resilience", "platform:hardware_and_scheduling"]  do |_, args|
   VERBOSE_LOGGING.info "platform" if check_verbose(args)
+  stdout_score("platform")
 end
 
 desc "Does the platform pass the K8s conformance tests?"
@@ -21,11 +22,11 @@ task "k8s_conformance" do |_, args|
     #TODO when in test mode --mode quick, prod mode no quick
     testrun = ""
     VERBOSE_LOGGING.info ENV["CRYSTAL_ENV"]? if check_verbose(args)
-    if ENV["CRYSTAL_ENV"]? == "TEST"
+    # if ENV["CRYSTAL_ENV"]? == "TEST"
       testrun = `#{sonobuoy} run --wait --mode quick`
-    else
-      testrun = `#{sonobuoy} run --wait`
-    end
+    # else
+    #   testrun = `#{sonobuoy} run --wait`
+    # end
     VERBOSE_LOGGING.info testrun if check_verbose(args)
 
     results = `results=$(#{sonobuoy} retrieve); #{sonobuoy} results $results` 

src/tasks/platform/resilience.cr

@@ -9,7 +9,7 @@ namespace "platform" do
     VERBOSE_LOGGING.info "resilience" if check_verbose(args)
     VERBOSE_LOGGING.debug "resilience args.raw: #{args.raw}" if check_verbose(args)
     VERBOSE_LOGGING.debug "resilience args.named: #{args.named}" if check_verbose(args)
-    stdout_score("resilience")
+    stdout_score("platform:resilience")
   end
 
   desc "Does the Platform recover the node and reschedule pods when a worker node fails"

src/tasks/utils/utils.cr

@@ -521,7 +521,8 @@ end
 
 def all_task_test_names
   result_items = points_yml.reduce([] of String) do |acc, x|
-    if x["name"].as_s == "default_scoring"
+    if x["name"].as_s == "default_scoring" ||
+        x["tags"].as_s.split(",").find{|x|x=="platform"}
       acc
     else
       acc << x["name"].as_s
@@ -585,7 +586,7 @@ end
 
 def stdout_score(test_name)
   total = total_points(test_name)
-  pretty_test_name = test_name.split("_").map(&.capitalize).join(" ")
+  pretty_test_name = test_name.split(/:|_/).map(&.capitalize).join(" ")
   test_log_msg = "#{pretty_test_name} final score: #{total} of #{total_max_points(test_name)}"
 
   if total > 0