Merge pull request #1910 from martin-mat/workload_tc_adapt

Workload tests adapted to centralized result logging

embedded_files/points.yml

@@ -8,24 +8,34 @@
   neutral: 0 
 
 - name: reasonable_image_size
+  emoji: "⚖👀"
   tags: [microservice, dynamic, workload, cert, normal]
 - name: specialized_init_system
+  emoji: "🚀"
   tags: [microservice, dynamic, workload]
 - name: reasonable_startup_time 
   tags: [microservice, dynamic, workload, cert, normal]
-- name: single_process_type 
+- name: single_process_type
+  emoji: "⚖👀"
   tags: [microservice, dynamic, workload, essential, cert]
   pass: 100
+- name: zombie_handled
+  emoji: "⚖👀"
+  tags: [microservice, dynamic, workload, normal]
 - name: service_discovery
+  emoji: "⚖👀"
   tags: [microservice, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: shared_database
+  emoji: "💾"
   tags: [microservice, dynamic, workload, cert, normal]
-- name: sig_term_handled 
+- name: sig_term_handled
+  emoji: "⚖👀"
   tags: [microservice, dynamic, workload, normal]
 
 - name: cni_compatible
+  emoji: "🔓🔑"
   tags: [compatibility, dynamic, workload, cert, normal]
 # - name: cni_spec 
 #  tags: compatibility, dynamic
@@ -41,23 +51,29 @@
 #- name: check_reaped 
 #  tags: state, dynamic, configuration
 
-- name: privileged 
+- name: privileged
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload]
   # required: true
-- name: privilege_escalation 
+- name: privilege_escalation
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 - name: symlink_file_system 
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 - name: application_credentials 
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 - name: host_network 
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 #- name: shells 
 #  tags: security, dynamic
 #- name: protected_access 
 #  tags: security, dynamic
 
-- name: increase_decrease_capacity 
+- name: increase_decrease_capacity
+  emoji: "📦📈📉"
   tags: [compatibility, dynamic, workload, essential, cert]
   pass: 100
 #- name: small_autoscaling 
@@ -67,28 +83,36 @@
 # - name: network_chaos 
 #   tags: resilience, dynamic, workload
 - name: pod_network_latency
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: pod_network_corruption
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: pod_network_duplication
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: pod_delete
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, normal]
 - name: pod_io_stress
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: pod_memory_hog
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, normal]
 - name: disk_fill
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, normal]
 - name: pod_dns_error
+  emoji: "🗡️💀♻"
   tags: [resilience, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
@@ -98,17 +122,22 @@
 #- name: versioned_helm_chart
 #  tags: configuration, dynamic, workload
 - name: versioned_tag
+  emoji: "🏷️"
   tags: [configuration, dynamic, workload]
 - name: ip_addresses
+  emoji: "📶🏃⏲️"
   pass: 0
   fail: -1 
   tags: [configuration, static, workload]
 - name: operator_installed
+  emoji: "⚖️👀"
   tags: [configuration, dynamic, workload, cert, bonus]
 - name: liveness
+  emoji: "⎈🧫"
   tags: [resilience, dynamic, workload, essential, cert]
   pass: 100
 - name: readiness
+  emoji: "⎈🧫"
   tags: [resilience, dynamic, workload, essential, cert]
   pass: 100
 #- name: no_volume_with_configuration
@@ -130,10 +159,12 @@
   tags: [configuration, dynamic, workload, essential, cert]
   pass: 100
 - name: secrets_used
+  emoji: "🧫"
   tags: [configuration, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: immutable_configmap
+  emoji: "⚖️"
   tags: [configuration, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0 
@@ -150,10 +181,13 @@
 #  tags: observability, dynamic, workload
 
 - name: helm_deploy
+  emoji: "⚙🛠️⬆☁"
   tags: [compatibility, dynamic, workload, cert, normal]
 - name: helm_chart_valid
+  emoji: "⎈📝☑"
   tags: [compatibility, dynamic, workload, cert, normal]
 - name: helm_chart_published
+  emoji: "⎈📦🌐"
   tags: [compatibility, dynamic, workload, cert, normal]
 
 # - name: chaos_network_loss 
@@ -164,21 +198,26 @@
 #   tags: resilience, dynamic, workload
 
 - name: volume_hostpath_not_found
+  emoji: "💾"
   tags: [state, dynamic, workload]
 - name: no_local_volume_configuration
+  emoji: "💾"
   tags: [state, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
-- name: elastic_volumes 
+- name: elastic_volumes
+  emoji: "🧫"
   tags: [state, dynamic, workload, cert, bonus]
   pass: 1
   fail: 0
 - name: database_persistence
+  emoji: "🧫"
   tags: [state, dynamic, workload]
   pass5: 5 
   pass3: 3 
   fail: -1
 - name: node_drain
+  emoji: "🗡️💀♻"
   tags: [state, dynamic, workload, essential, cert]
   pass: 100
 
@@ -207,46 +246,57 @@
   tags: ["platform", "platform:security", "dynamic"]
 
 - name: service_account_mapping
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 
 - name: privileged_containers
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, essential, cert]
   pass: 100 
 
 - name: non_root_containers
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, essential, cert]
   pass: 100
 
 - name: host_pid_ipc_privileges
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 
 - name: linux_hardening
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 
 
 - name: resource_policies
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, essential]
   pass: 100
 
 - name: immutable_file_systems
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 
 
 - name: hostpath_mounts
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, essential, cert]
   pass: 100
 
 - name: ingress_egress_blocked 
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, bonus]
   pass: 1 
   fail: 0 
 
 - name: insecure_capabilities
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 
 - name: sysctls
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 
 - name: log_output
@@ -274,29 +324,36 @@
   pass: 1 
   fail: 0 
 - name: alpha_k8s_apis
+  emoji: "⭕🔍"
   tags: [configuration, dynamic, workload]
 
 - name: container_sock_mounts
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, essential, cert]
   pass: 100
 
 - name: require_labels
+  emoji: "🏷️"
   tags: [configuration, dynamic, workload, cert, normal]
 
 - name: helm_tiller
   tags: ["platform", "platform:security", "dynamic"]
 
 - name: external_ips
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, cert, normal]
 
 - name: selinux_options
+  emoji: "🔓🔑"
   tags: [security, dynamic, workload, essential, cert]
   pass: 100
 
 - name: default_namespace
+  emoji: "🏷️"
   tags: [configuration, dynamic, workload, cert, normal]
 
 - name: latest_tag
+  emoji: "🏷️"
   tags: [configuration, dynamic, workload, essential, cert]
   pass: 100
 

spec/utils/cnf_manager_spec.cr

@@ -146,7 +146,7 @@ describe "SampleUtils" do
 
   it "'CNFManager::Points.all_task_test_names' should return all tasks names", tags: ["points"] do
     CNFManager::Points.clean_results_yml
-		tags = ["alpha_k8s_apis", "application_credentials", "cni_compatible", "container_sock_mounts", "database_persistence", "default_namespace", "disk_fill", "elastic_volumes", "external_ips", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "helm_chart_published", "helm_chart_valid", "helm_deploy", "host_network", "host_pid_ipc_privileges", "hostpath_mounts", "hostport_not_used", "immutable_configmap", "immutable_file_systems", "increase_decrease_capacity", "ingress_egress_blocked", "insecure_capabilities", "ip_addresses", "latest_tag", "linux_hardening", "liveness", "log_output", "no_local_volume_configuration", "node_drain", "nodeport_not_used", "non_root_containers", "open_metrics", "operator_installed", "oran_e2_connection", "pod_delete", "pod_dns_error", "pod_io_stress", "pod_memory_hog", "pod_network_corruption", "pod_network_duplication", "pod_network_latency", "privilege_escalation", "privileged", "privileged_containers", "prometheus_traffic", "readiness", "reasonable_image_size", "reasonable_startup_time", "require_labels", "resource_policies", "rollback", "rolling_downgrade", "rolling_update", "rolling_version_change", "routed_logs", "secrets_used", "selinux_options", "service_account_mapping", "service_discovery", "shared_database", "sig_term_handled", "single_process_type", "smf_upf_heartbeat", "specialized_init_system", "suci_enabled", "symlink_file_system", "sysctls", "tracing", "versioned_tag", "volume_hostpath_not_found"]
+		tags = ["alpha_k8s_apis", "application_credentials", "cni_compatible", "container_sock_mounts", "database_persistence", "default_namespace", "disk_fill", "elastic_volumes", "external_ips", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "helm_chart_published", "helm_chart_valid", "helm_deploy", "host_network", "host_pid_ipc_privileges", "hostpath_mounts", "hostport_not_used", "immutable_configmap", "immutable_file_systems", "increase_decrease_capacity", "ingress_egress_blocked", "insecure_capabilities", "ip_addresses", "latest_tag", "linux_hardening", "liveness", "log_output", "no_local_volume_configuration", "node_drain", "nodeport_not_used", "non_root_containers", "open_metrics", "operator_installed", "oran_e2_connection", "pod_delete", "pod_dns_error", "pod_io_stress", "pod_memory_hog", "pod_network_corruption", "pod_network_duplication", "pod_network_latency", "privilege_escalation", "privileged", "privileged_containers", "prometheus_traffic", "readiness", "reasonable_image_size", "reasonable_startup_time", "require_labels", "resource_policies", "rollback", "rolling_downgrade", "rolling_update", "rolling_version_change", "routed_logs", "secrets_used", "selinux_options", "service_account_mapping", "service_discovery", "shared_database", "sig_term_handled", "single_process_type", "smf_upf_heartbeat", "specialized_init_system", "suci_enabled", "symlink_file_system", "sysctls", "tracing", "versioned_tag", "volume_hostpath_not_found", "zombie_handled"]
     (CNFManager::Points.all_task_test_names()).sort.should eq(tags.sort)
   end
 
@@ -349,7 +349,7 @@ describe "SampleUtils" do
   it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, when a cnf has a valid config file yml", tags: ["validate_config"]  do
     response_s = `./cnf-testsuite validate_config cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml`
     $?.success?.should be_true
-    (/PASSED: CNF configuration validated/ =~ response_s).should_not be_nil
+    (/CNF configuration validated/ =~ response_s).should_not be_nil
   end
 
 
@@ -375,7 +375,7 @@ describe "SampleUtils" do
     LOGGING.debug "validate_config resp: #{response_s}"
     # (/WARNING: Unmapped cnf_testsuite.yml keys. Please add them to the validator/ =~ response_s).should_not be_nil
     # (/WARNING: helm_repository is unset or has unmapped subkeys. Please update your cnf_testsuite.yml/ =~ response_s).should_not be_nil
-    (/PASSED: CNF configuration validated/ =~ response_s).should_not be_nil
+    (/CNF configuration validated/ =~ response_s).should_not be_nil
   end
 
 
@@ -387,7 +387,7 @@ describe "SampleUtils" do
       testsuite_yml = "sample-cnfs/#{dir}/cnf-testsuite.yml"
       response_s = `./cnf-testsuite validate_config cnf-config=#{testsuite_yml}`
       Log.info { "\n #{testsuite_yml}: #{response_s}" }
-      (/PASSED: CNF configuration validated/ =~ response_s).should_not be_nil
+      (/CNF configuration validated/ =~ response_s).should_not be_nil
     end
   end
 
@@ -398,10 +398,10 @@ describe "SampleUtils" do
     dir_list.each do |dir|
       testsuite_yml = "example-cnfs/#{dir}/cnf-testsuite.yml"
       response_s = `./cnf-testsuite validate_config cnf-config=#{testsuite_yml}`
-      if (/FAILED: Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file/ =~ response_s)
+      if (/Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file/ =~ response_s)
         LOGGING.info "\n #{testsuite_yml}: #{response_s}"
       end
-      (/PASSED: CNF configuration validated/ =~ response_s).should_not be_nil
+      (/CNF configuration validated/ =~ response_s).should_not be_nil
     end
   end
 

spec/workload/operator_spec.cr

@@ -85,7 +85,7 @@ describe "Operator" do
       $?.success?.should be_true
       resp = `./cnf-testsuite -l info operator_installed`
       Log.info { "#{resp}" }
-      (/NA: No Operators Found/ =~ resp).should_not be_nil
+      (/N\/A: No Operators Found/ =~ resp).should_not be_nil
     ensure
       LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns`
       $?.success?.should be_true

src/tasks/utils/points.cr

@@ -556,7 +556,7 @@ module CNFManager
         Log.debug { "task #{task} emoji: #{md["emoji"]?}" }
         resp = md["emoji"]
       else
-        resp = [] of String
+        resp = ""
       end
     end