diff --git a/embedded_files/points.yml b/embedded_files/points.yml index 29ea241ed..5b6e238bd 100644 --- a/embedded_files/points.yml +++ b/embedded_files/points.yml @@ -12,7 +12,8 @@ tags: [microservice, dynamic, workload, cert, normal] - name: specialized_init_system emoji: "🚀" - tags: [microservice, dynamic, workload] + tags: [microservice, dynamic, workload, essential, cert] + pass: 100 - name: reasonable_startup_time tags: [microservice, dynamic, workload, cert, normal] - name: single_process_type @@ -21,7 +22,8 @@ pass: 100 - name: zombie_handled emoji: "⚖👀" - tags: [microservice, dynamic, workload, normal] + tags: [microservice, dynamic, workload, essential, cert] + pass: 100 - name: service_discovery emoji: "⚖👀" tags: [microservice, dynamic, workload, cert, bonus] @@ -32,8 +34,8 @@ tags: [microservice, dynamic, workload, cert, normal] - name: sig_term_handled emoji: "⚖👀" - tags: [microservice, dynamic, workload, normal] - + tags: [microservice, dynamic, workload, essential, cert] + pass: 100 - name: cni_compatible emoji: "🔓🔑" tags: [compatibility, dynamic, workload, cert, normal] @@ -113,8 +115,8 @@ tags: [resilience, dynamic, workload, cert, normal] - name: pod_dns_error emoji: "🗡️💀♻" - tags: [resilience, dynamic, workload, cert, bonus] - pass: 1 + tags: [resilience, dynamic, workload, essential, cert] + pass: 100 fail: 0 #- name: external_retry # tags: scalability, dynamic, workload @@ -199,7 +201,8 @@ - name: volume_hostpath_not_found emoji: "💾" - tags: [state, dynamic, workload] + tags: [state, dynamic, workload, essential, cert] + pass: 100 - name: no_local_volume_configuration emoji: "💾" tags: [state, dynamic, workload, cert, bonus] diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index 5898c5cff..e4f9c92ef 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -449,7 +449,7 @@ describe "SampleUtils" do Log.info { install } response_s = `./cnf-testsuite cert_microservice` Log.info { response_s} - (/of 3 tests passed/ =~ response_s).should_not be_nil + (/of 6 tests passed/ =~ response_s).should_not be_nil Log.info { response_s } ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged` } diff --git a/src/tasks/cert/cert.cr b/src/tasks/cert/cert.cr index 31ce5cae5..870a35385 100644 --- a/src/tasks/cert/cert.cr +++ b/src/tasks/cert/cert.cr @@ -20,7 +20,7 @@ task "cert", ["version", "cert_compatibility", "cert_state", "cert_security", "c essential_max_passed = CNFManager::Points.total_max_passed("essential") stdout_success " - #{total_passed} of #{max_passed} total tests passed" - if ESSENTIAL_PASSING_SCORE >= 1000 + if essential_total_passed >= ESSENTIAL_PASSED_THRESHOLD stdout_success " - #{essential_total_passed} of #{essential_max_passed} essential tests passed" else stdout_failure "FAILED: #{essential_total_passed} of #{essential_max_passed} essential tests passed" diff --git a/src/tasks/cert/cert_microservice.cr b/src/tasks/cert/cert_microservice.cr index d68318f44..80cd760ed 100644 --- a/src/tasks/cert/cert_microservice.cr +++ b/src/tasks/cert/cert_microservice.cr @@ -9,7 +9,7 @@ require "halite" require "totem" desc "The CNF test suite checks to see if CNFs follows microservice principles" -task "cert_microservice", ["cert_microservice_title","reasonable_image_size", "reasonable_startup_time", "single_process_type", "service_discovery", "shared_database"] do |_, args| +task "cert_microservice", ["cert_microservice_title","reasonable_image_size", "reasonable_startup_time", "single_process_type", "service_discovery", "shared_database", "zombie_handled", "sig_term_handled", "specialized_init_system"] do |_, args| # task "cert_microservice", ["cert_microservice_title", "reasonable_image_size", "reasonable_startup_time", "service_discovery"] do |_, args| # stdout_score("microservice") stdout_score(["microservice", "cert"], "microservice") diff --git a/src/tasks/cert/cert_state.cr b/src/tasks/cert/cert_state.cr index 0e3c70a99..560a3c432 100644 --- a/src/tasks/cert/cert_state.cr +++ b/src/tasks/cert/cert_state.cr @@ -7,7 +7,7 @@ require "../utils/utils.cr" require "kubectl_client" desc "The CNF test suite checks if state is stored in a custom resource definition or a separate database (e.g. etcd) rather than requiring local storage. It also checks to see if state is resilient to node failure" -task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes", "node_drain"] do |_, args| +task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes", "node_drain", "volume_hostpath_not_found"] do |_, args| # task "cert_state", ["cert_state_title", "no_local_volume_configuration", "elastic_volumes" ] do |_, args| # stdout_score("state") stdout_score(["state", "cert"], "state") diff --git a/src/tasks/constants.cr b/src/tasks/constants.cr index 88a98551c..5cc0f9571 100644 --- a/src/tasks/constants.cr +++ b/src/tasks/constants.cr @@ -1,6 +1,6 @@ require "./utils/embedded_file_manager.cr" -ESSENTIAL_PASSING_SCORE = 1000 +ESSENTIAL_PASSED_THRESHOLD = 15 CNF_DIR = "cnfs" CONFIG_FILE = "cnf-testsuite.yml" BASE_CONFIG = "./config.yml"