From 91bc202e78c474ecea0b45844ac24849c0907fc1 Mon Sep 17 00:00:00 2001 From: Akash Date: Tue, 24 Oct 2023 11:00:51 +0000 Subject: [PATCH 1/2] Add two more copies of coredns CNF to be able to test node_drain Signed-off-by: denverwilliams --- sample-cnfs/sample-coredns-cnf3/README.md | 39 +++++++++++++++++++ .../sample-coredns-cnf3/cnf-testsuite.yml | 9 +++++ sample-cnfs/sample-coredns-cnf4/README.md | 39 +++++++++++++++++++ .../sample-coredns-cnf4/cnf-testsuite.yml | 9 +++++ 4 files changed, 96 insertions(+) create mode 100644 sample-cnfs/sample-coredns-cnf3/README.md create mode 100644 sample-cnfs/sample-coredns-cnf3/cnf-testsuite.yml create mode 100644 sample-cnfs/sample-coredns-cnf4/README.md create mode 100644 sample-cnfs/sample-coredns-cnf4/cnf-testsuite.yml diff --git a/sample-cnfs/sample-coredns-cnf3/README.md b/sample-cnfs/sample-coredns-cnf3/README.md new file mode 100644 index 000000000..359c3868f --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf3/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-testsuite config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-testsuite.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample-coredns-cnf3/cnf-testsuite.yml b/sample-cnfs/sample-coredns-cnf3/cnf-testsuite.yml new file mode 100644 index 000000000..b1f5dd6ef --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf3/cnf-testsuite.yml @@ -0,0 +1,9 @@ +--- +release_name: coredns3 +service_name: coredns-coredns +helm_repository: + name: stable + repo_url: https://cncf.gitlab.io/stable +helm_chart: stable/coredns +helm_install_namespace: cnfspace3 +allowlist_helm_chart_container_names: [falco, node-cache, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/sample-cnfs/sample-coredns-cnf4/README.md b/sample-cnfs/sample-coredns-cnf4/README.md new file mode 100644 index 000000000..359c3868f --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf4/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-testsuite config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-testsuite.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample-coredns-cnf4/cnf-testsuite.yml b/sample-cnfs/sample-coredns-cnf4/cnf-testsuite.yml new file mode 100644 index 000000000..728c0302a --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf4/cnf-testsuite.yml @@ -0,0 +1,9 @@ +--- +release_name: coredns4 +service_name: coredns-coredns +helm_repository: + name: stable + repo_url: https://cncf.gitlab.io/stable +helm_chart: stable/coredns +helm_install_namespace: cnfspace4 +allowlist_helm_chart_container_names: [falco, node-cache, nginx, coredns, calico-node, kube-proxy, nginx-proxy] From 82853d494ff3373108de73b421bdaf9263ab6554 Mon Sep 17 00:00:00 2001 From: Akash Date: Tue, 24 Oct 2023 11:18:21 +0000 Subject: [PATCH 2/2] LitmusManager.wait_for_test: Increase wait_count to 30min * Update node_drain chaos experiment name to use Random::Secure.hex/1 instead of Random.rand/1 * Output node_drain experiment name and test name to debug log. * Output command to inspect chaos experiment if node_drain fails for CNF * Output chaos experiment resource/result to debug log if test fails for CNF Signed-off-by: Akash Manohar --- src/tasks/litmus_setup.cr | 15 ++++++++++----- src/tasks/workload/state.cr | 4 +++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/tasks/litmus_setup.cr b/src/tasks/litmus_setup.cr index 6dd12b4e9..750a284bc 100644 --- a/src/tasks/litmus_setup.cr +++ b/src/tasks/litmus_setup.cr @@ -99,7 +99,7 @@ module LitmusManager Log.for("wait_for_test").info { "Checking experiment status #{experimentStatus_cmd}" } if check_verbose(args) ## Wait for completion of chaosengine which indicates the completion of chaos - until (status_code == 0 && experimentStatus == "Completed") || wait_count >= retry + until (status_code == 0 && experimentStatus == "Completed") || wait_count >= 1800 sleep delay experimentStatus_cmd = "kubectl get chaosengine.litmuschaos.io #{test_name} -n #{namespace} -o jsonpath='{.status.experiments[0].status}'" Log.for("wait_for_test").info { "Checking experiment status #{experimentStatus_cmd}" } if check_verbose(args) @@ -135,17 +135,22 @@ module LitmusManager ## check_chaos_verdict will check the verdict of chaosexperiment def self.check_chaos_verdict(chaos_result_name, chaos_experiment_name, args, namespace : String = "default") : Bool verdict_cmd = "kubectl get chaosresults.litmuschaos.io #{chaos_result_name} -n #{namespace} -o jsonpath='{.status.experimentStatus.verdict}'" - Log.for("check_chaos_verdict").info { "Checking experiment verdict #{verdict_cmd}" } if check_verbose(args) + Log.for("LitmusManager.check_chaos_verdict").debug { "Checking experiment verdict with command: #{verdict_cmd}" } status_code = Process.run("#{verdict_cmd}", shell: true, output: verdict_response = IO::Memory.new, error: stderr = IO::Memory.new).exit_status - Log.for("check_chaos_verdict").info { "status_code: #{status_code}" } if check_verbose(args) - Log.for("check_chaos_verdict").info { "verdict: #{verdict_response.to_s}" } if check_verbose(args) + Log.for("LitmusManager.check_chaos_verdict").debug { "status_code: #{status_code}; verdict: #{verdict_response.to_s}" } verdict = verdict_response.to_s emoji_test_failed= "🗡️💀♻️" if verdict == "Pass" return true else - Log.info {"#{chaos_experiment_name} chaos test failed: #{chaos_result_name}, verdict: #{verdict}"} + Log.for("LitmusManager.check_chaos_verdict#details").debug do + verdict_details_cmd = "kubectl get chaosresults.litmuschaos.io #{chaos_result_name} -n #{namespace} -o json" + status_code = Process.run("#{verdict_details_cmd}", shell: true, output: verdict_details_response = IO::Memory.new, error: stderr = IO::Memory.new).exit_status + "#{verdict_details_response.to_s}" + end + + Log.for("LitmusManager.check_chaos_verdict").info {"#{chaos_experiment_name} chaos test failed: #{chaos_result_name}, verdict: #{verdict}"} return false end end diff --git a/src/tasks/workload/state.cr b/src/tasks/workload/state.cr index 519ba7dd3..a287b0695 100644 --- a/src/tasks/workload/state.cr +++ b/src/tasks/workload/state.cr @@ -330,7 +330,7 @@ task "node_drain", ["install_litmus"] do |t, args| chaos_experiment_name = "node-drain" total_chaos_duration = "90" - test_name = "#{resource["name"]}-#{Random.rand(99)}" + test_name = "#{resource["name"]}-#{Random::Secure.hex(4)}" chaos_result_name = "#{test_name}-#{chaos_experiment_name}" template = ChaosTemplates::NodeDrain.new( @@ -342,6 +342,8 @@ task "node_drain", ["install_litmus"] do |t, args| total_chaos_duration, app_nodeName ).to_s + Log.for("node_drain").info { "Chaos test name: #{test_name}; Experiment name: #{chaos_experiment_name}; Label #{deployment_label}=#{deployment_label_value}; namespace: #{app_namespace}" } + File.write("#{destination_cnf_dir}/#{chaos_experiment_name}-chaosengine.yml", template) KubectlClient::Apply.file("#{destination_cnf_dir}/#{chaos_experiment_name}-chaosengine.yml") LitmusManager.wait_for_test(test_name,chaos_experiment_name,total_chaos_duration,args, namespace: app_namespace)