Adapt cert tests for CNTI certification V2.0

Ref: #1932 Signed-off-by: Martin Matyas <[email protected]>
cnti-testcatalog · Apr 8, 2024 · 8a6d71c · 8a6d71c
1 parent daccafa
commit 8a6d71c
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 12 deletions.
diff --git a/embedded_files/points.yml b/embedded_files/points.yml
@@ -12,7 +12,8 @@
   tags: [microservice, dynamic, workload, cert, normal]
 - name: specialized_init_system
   emoji: "🚀"
-  tags: [microservice, dynamic, workload]
+  tags: [microservice, dynamic, workload, essential, cert]
+  pass: 100
 - name: reasonable_startup_time 
   tags: [microservice, dynamic, workload, cert, normal]
 - name: single_process_type
@@ -21,7 +22,8 @@
   pass: 100
 - name: zombie_handled
   emoji: "⚖👀"
-  tags: [microservice, dynamic, workload, normal]
+  tags: [microservice, dynamic, workload, essential, cert]
+  pass: 100
 - name: service_discovery
   emoji: "⚖👀"
   tags: [microservice, dynamic, workload, cert, bonus]
@@ -32,8 +34,8 @@
   tags: [microservice, dynamic, workload, cert, normal]
 - name: sig_term_handled
   emoji: "⚖👀"
-  tags: [microservice, dynamic, workload, normal]
-
+  tags: [microservice, dynamic, workload, essential, cert]
+  pass: 100
 - name: cni_compatible
   emoji: "🔓🔑"
   tags: [compatibility, dynamic, workload, cert, normal]
@@ -113,8 +115,8 @@
   tags: [resilience, dynamic, workload, cert, normal]
 - name: pod_dns_error
   emoji: "🗡️💀♻"
-  tags: [resilience, dynamic, workload, cert, bonus]
-  pass: 1
+  tags: [resilience, dynamic, workload, essential, cert]
+  pass: 100
   fail: 0
 #- name: external_retry 
 #  tags: scalability, dynamic, workload
@@ -199,7 +201,8 @@
 
 - name: volume_hostpath_not_found
   emoji: "💾"
-  tags: [state, dynamic, workload]
+  tags: [state, dynamic, workload, essential, cert]
+  pass: 100
 - name: no_local_volume_configuration
   emoji: "💾"
   tags: [state, dynamic, workload, cert, bonus]

diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr
@@ -449,7 +449,7 @@ describe "SampleUtils" do
       Log.info { install }
       response_s = `./cnf-testsuite cert_microservice`
       Log.info { response_s}
-      (/of 3 tests passed/ =~ response_s).should_not be_nil
+      (/of 6 tests passed/ =~ response_s).should_not be_nil
       Log.info { response_s }
     ensure
       Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged` }

diff --git a/src/tasks/cert/cert.cr b/src/tasks/cert/cert.cr
@@ -20,7 +20,7 @@ task "cert", ["version", "cert_compatibility", "cert_state", "cert_security", "c
   essential_max_passed = CNFManager::Points.total_max_passed("essential")
   stdout_success "  - #{total_passed} of #{max_passed} total tests passed"
 
-  if ESSENTIAL_PASSING_SCORE >= 1000 
+  if essential_total_passed >= ESSENTIAL_PASSED_THRESHOLD
     stdout_success "  - #{essential_total_passed} of #{essential_max_passed} essential tests passed"
   else
     stdout_failure "FAILED: #{essential_total_passed} of #{essential_max_passed} essential tests passed"

diff --git a/src/tasks/cert/cert_microservice.cr b/src/tasks/cert/cert_microservice.cr
@@ -9,7 +9,7 @@ require "halite"
 require "totem"
 
 desc "The CNF test suite checks to see if CNFs follows microservice principles"
-task "cert_microservice", ["cert_microservice_title","reasonable_image_size", "reasonable_startup_time", "single_process_type", "service_discovery", "shared_database"] do |_, args|
+task "cert_microservice", ["cert_microservice_title","reasonable_image_size", "reasonable_startup_time", "single_process_type", "service_discovery", "shared_database", "zombie_handled", "sig_term_handled", "specialized_init_system"] do |_, args|
 # task "cert_microservice", ["cert_microservice_title", "reasonable_image_size", "reasonable_startup_time", "service_discovery"] do |_, args|
   # stdout_score("microservice")
   stdout_score(["microservice", "cert"], "microservice")

diff --git a/src/tasks/cert/cert_state.cr b/src/tasks/cert/cert_state.cr
@@ -7,7 +7,7 @@ require "../utils/utils.cr"
 require "kubectl_client"
 
 desc "The CNF test suite checks if state is stored in a custom resource definition or a separate database (e.g. etcd) rather than requiring local storage.  It also checks to see if state is resilient to node failure"
-task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes", "node_drain"] do |_, args|
+task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes", "node_drain", "volume_hostpath_not_found"] do |_, args|
 # task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes" ] do |_, args|
   # stdout_score("state")
   stdout_score(["state", "cert"], "state")

diff --git a/src/tasks/constants.cr b/src/tasks/constants.cr
@@ -1,6 +1,6 @@
 require "./utils/embedded_file_manager.cr"
 
-ESSENTIAL_PASSING_SCORE = 1000
+ESSENTIAL_PASSED_THRESHOLD = 15
 CNF_DIR = "cnfs"
 CONFIG_FILE = "cnf-testsuite.yml"
 BASE_CONFIG = "./config.yml"