Modularize implementation of tests #1861

embedded_files/points.yml

@@ -250,21 +250,26 @@
   tags: [security, dynamic, workload, cert, normal]
 
 - name: log_output
+  emoji: "📶☠️"
   tags: [observability, dynamic, workload, essential, cert]
   pass: 100
-- name: prometheus_traffic 
+- name: prometheus_traffic
+  emoji: "📶☠️"
   tags: [observability, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 
-- name: open_metrics 
+- name: open_metrics
+  emoji: "📶☠️"
   tags: [observability, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 
-- name: routed_logs 
+- name: routed_logs
+  emoji: "📶☠️"
   tags: [observability, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 
-- name: tracing 
+- name: tracing
+  emoji: "⎈🚀"
   tags: [observability, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 

spec/utils/cnf_manager_spec.cr

@@ -66,7 +66,7 @@ describe "SampleUtils" do
 
   it  "'task_points(, skipped)' should return the amount of points for a skipped test", tags: ["points"]  do
     # default
-    (CNFManager::Points.task_points("liveness", CNFManager::Points::Results::ResultStatus::Skipped)).should eq(0)
+    (CNFManager::Points.task_points("liveness", CNFManager::ResultStatus::Skipped)).should eq(0)
   end
 
   # todo add mock test for testing require tests (privilege mode is no longer required)

spec/utils/utils_spec.cr

@@ -67,7 +67,7 @@ describe "Utils" do
     end
     (yaml["items"].as_a.find {|x| 
       x["name"] == "ip_addresses" && 
-        x["points"] == CNFManager::Points.task_points("ip_addresses", CNFManager::Points::Results::ResultStatus::Skipped)
+        x["points"] == CNFManager::Points.task_points("ip_addresses", CNFManager::ResultStatus::Skipped)
     }).should be_truthy
 
     (yaml["items"].as_a.find {|x| x["name"] == "ip_addresses" && x["points"] == 0 }).should be_truthy

src/tasks/utils/points.cr

@@ -4,8 +4,37 @@ require "./types/cnf_testsuite_yml_type.cr"
 require "helm"
 require "uuid"
 
+
 module CNFManager
 
+  enum ResultStatus
+    Passed
+    Failed
+    Skipped
+    NA
+    Neutral
+    Pass5
+    Pass3
+
+    def to_basic()
+      case self
+      when Pass5, Pass3
+        ret = CNFManager::ResultStatus::Passed
+      when Neutral
+        ret = CNFManager::ResultStatus::Failed
+      else
+        ret = self
+      end
+    end
+  end
+
+  struct TestcaseResult
+    property state, result_message
+
+    def initialize(@state : CNFManager::ResultStatus, @result_message : String|Nil = nil)
+    end
+  end
+
   module Points
     class Results
 
@@ -101,14 +130,14 @@ module CNFManager
     end
 
     # Returns what the potential points should be (for a points type) in order to assign those points to a task
-    def self.task_points(task, status : CNFManager::Points::Results::ResultStatus = CNFManager::Points::Results::ResultStatus::Passed)
+    def self.task_points(task, status : CNFManager::ResultStatus = CNFManager::ResultStatus::Passed)
       #todo replace case statement with dynamic point call
       case status
-      when CNFManager::Points::Results::ResultStatus::Passed
+      when CNFManager::ResultStatus::Passed
         resp = CNFManager::Points.task_points(task, true)
-      when CNFManager::Points::Results::ResultStatus::Failed
+      when CNFManager::ResultStatus::Failed
         resp = CNFManager::Points.task_points(task, false)
-      when CNFManager::Points::Results::ResultStatus::Skipped
+      when CNFManager::ResultStatus::Skipped
         resp = dynamic_task_points(task, "skipped")
         # field_name = "skipped"
         # points =points_yml.find {|x| x["name"] == task}
@@ -119,7 +148,7 @@ module CNFManager
         #   points =points_yml.find {|x| x["name"] == "default_scoring"}
         #   resp = points[field_name].as_i if points
         # end
-      when CNFManager::Points::Results::ResultStatus::NA
+      when CNFManager::ResultStatus::NA
         resp = dynamic_task_points(task, "na")
         # field_name = "na"
         # points =points_yml.find {|x| x["name"] == task}
@@ -520,6 +549,17 @@ module CNFManager
       end
     end
 
+    def self.emoji_by_task(task)
+      md = points_yml.find {|x| x["name"] == task}
+      Log.warn { "****Warning**** task #{task} not found in points.yml".colorize(:yellow) } unless md
+      if md && md["emoji"]?
+        Log.debug { "task #{task} emoji: #{md["emoji"]?}" }
+        resp = md["emoji"]
+      else
+        resp = [] of String
+      end
+    end
+
     def self.tags_by_task(task)
       points =points_yml.find {|x| x["name"] == task}
       Log.warn { "****Warning**** task #{task} not found in points.yml".colorize(:yellow) } unless points
@@ -554,6 +594,17 @@ module CNFManager
       task_type
     end
 
+    def self.task_emoji_by_task(task)
+      case self.task_type_by_task(task)
+      when "essential"
+        "🏆"
+      when "bonus"
+        "✨"
+      else
+        ""
+      end
+    end
+
     def self.all_result_test_names(results_file)
       results = File.open(results_file) do |f|
         YAML.parse(f)

src/tasks/utils/task.cr

@@ -23,7 +23,7 @@ module CNFManager
       end
     end
 
-    def self.task_runner(args, check_cnf_installed=true, &block : Sam::Args, CNFManager::Config -> String | Colorize::Object(String) | Nil)
+    def self.task_runner(args, task : Sam::Task|Nil=nil, check_cnf_installed=true, &block : Sam::Args, CNFManager::Config -> String | Colorize::Object(String) | CNFManager::TestcaseResult | Nil)
       LOGGING.info("task_runner args: #{args.inspect}")
 
       CNFManager::Points::Results.ensure_results_file!
@@ -33,14 +33,14 @@ module CNFManager
       end
 
       if check_cnf_config(args)
-        single_task_runner(args, &block)
+        single_task_runner(args, task, &block)
       else
-        all_cnfs_task_runner(args, &block)
+        all_cnfs_task_runner(args, task, &block)
       end
     end
 
     # TODO give example for calling
-    def self.all_cnfs_task_runner(args, &block : Sam::Args, CNFManager::Config  -> String | Colorize::Object(String) | Nil)
+    def self.all_cnfs_task_runner(args, task : Sam::Task|Nil=nil, &block : Sam::Args, CNFManager::Config  -> String | Colorize::Object(String) | CNFManager::TestcaseResult | Nil)
       cnf_configs = CNFManager.cnf_config_list(silent: true)
       Log.info { "CNF configs found: #{cnf_configs.size}" }
 
@@ -51,13 +51,13 @@ module CNFManager
         cnf_configs.map do |x|
           new_args = Sam::Args.new(args.named, args.raw)
           new_args.named["cnf-config"] = x
-          single_task_runner(new_args, &block)
+          single_task_runner(new_args, task, &block)
         end
       end
     end
 
     # TODO give example for calling
-    def self.single_task_runner(args, &block : Sam::Args, CNFManager::Config -> String | Colorize::Object(String) | Nil)
+    def self.single_task_runner(args, task : Sam::Task|Nil=nil, &block : Sam::Args, CNFManager::Config -> String | Colorize::Object(String) | CNFManager::TestcaseResult | Nil)
       LOGGING.debug("single_task_runner args: #{args.inspect}")
 
       begin
@@ -110,7 +110,16 @@ module CNFManager
                                                           },
                                             image_registry_fqdns: Hash(String, String).new} )
         end
+        test_start_time = Time.utc
+        if task
+          test_name = task.as(Sam::Task).name.as(String)
+          Log.for(test_name).info { "Starting test" }
+          Log.for(test_name).debug { "cnf_config: #{config}" }
+        end
         ret = yield args, config
+        if ret.is_a?(CNFManager::TestcaseResult)
+          upsert_decorated_task(test_name, ret.state, ret.result_message, test_start_time)
+        end
         #todo lax mode, never returns 1
         if args.raw.includes? "strict"
           if CNFManager::Points.failed_required_tasks.size > 0

src/tasks/utils/utils.cr

@@ -289,6 +289,21 @@ def update_yml(yml_file, top_level_key, value)
   end
 end
 
+def upsert_decorated_task(task, status : CNFManager::ResultStatus, message, start_time)
+  tc_emoji = CNFManager::Points.emoji_by_task(task)
+  cat_emoji = CNFManager::Points.task_emoji_by_task(task)
+  case status.to_basic
+  when CNFManager::ResultStatus::Passed
+    upsert_passed_task(task, "✔️  #{cat_emoji}PASSED: #{message} #{tc_emoji}", start_time)
+  when CNFManager::ResultStatus::Failed
+    upsert_failed_task(task, "✖️  #{cat_emoji}FAILED: #{message} #{tc_emoji}", start_time)
+  when CNFManager::ResultStatus::Skipped
+    upsert_skipped_task(task, "⏭️  #{cat_emoji}SKIPPED: #{message} #{tc_emoji}", start_time)
+  when CNFManager::ResultStatus::NA
+    upsert_na_task(task, "⏭️  #{cat_emoji}N/A: #{message} #{tc_emoji}", start_time)
+  end
+end
+
 def upsert_failed_task(task, message, start_time)
  CNFManager::Points.upsert_task(task, FAILED, CNFManager::Points.task_points(task, false), start_time)
   stdout_failure message
@@ -302,18 +317,18 @@ def upsert_passed_task(task, message, start_time)
 end
 
 def upsert_skipped_task(task, message, start_time)
- CNFManager::Points.upsert_task(task, SKIPPED, CNFManager::Points.task_points(task, CNFManager::Points::Results::ResultStatus::Skipped), start_time)
+ CNFManager::Points.upsert_task(task, SKIPPED, CNFManager::Points.task_points(task, CNFManager::ResultStatus::Skipped), start_time)
   stdout_warning message
   message
 end
 
 def upsert_na_task(task, message, start_time)
- CNFManager::Points.upsert_task(task, NA, CNFManager::Points.task_points(task, CNFManager::Points::Results::ResultStatus::NA), start_time)
+ CNFManager::Points.upsert_task(task, NA, CNFManager::Points.task_points(task, CNFManager::ResultStatus::NA), start_time)
   stdout_warning message
   message
 end
 
-def upsert_dynamic_task(task, status : CNFManager::Points::Results::ResultStatus, message, start_time)
+def upsert_dynamic_task(task, status : CNFManager::ResultStatus, message, start_time)
   CNFManager::Points.upsert_task(task, status.to_s.downcase, CNFManager::Points.task_points(task, status), start_time)
   case status.to_s.downcase 
   when /pass/