Merge pull request #1846 from cncf/1840_core_validator

Add 5G Core Validator Test #1840

sample-cnfs/sample_open5gs/cnf-testsuite.yml

@@ -3,7 +3,9 @@ helm_directory: open5gs
 release_name: open5gs
 allowlist_helm_chart_container_names: []
 #optional 5gcore tag
-core_label: app.kubernetes.io/name=amf
+amf_label: app.kubernetes.io/name=amf
+smf_label: app.kubernetes.io/name=smf
+upf_label: app.kubernetes.io/name=upf
 amf_pod_name: open5gs-amf-ngap
 mmc: '999'
 mnc: '70'

sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml

@@ -3,7 +3,7 @@ helm_directory: open5gs
 release_name: open5gs
 allowlist_helm_chart_container_names: []
 #optional 5gcore tag
-core_label: app.kubernetes.io/name=amf
+amf_label: app.kubernetes.io/name=amf
 amf_pod_name: open5gs-amf-ngap
 mmc: '999'
 mnc: '70'

shard.lock

@@ -54,7 +54,7 @@ shards:
 
   kubectl_client:
     git: https://github.com/cnf-testsuite/kubectl_client.git
-    version: 1.0.4
+    version: 1.0.5
 
   popcorn:
     git: https://github.com/icyleaf/popcorn.git
@@ -66,15 +66,15 @@ shards:
 
   readline:
     git: https://github.com/crystal-lang/crystal-readline.git
-    version: 0.1.1+git.commit.add6679775d59c164e2db04f5116557180d04ad9
+    version: 0.1.1+git.commit.69ecf33d7cad5568d7d19333510cfd9d17cb1bbd
 
   release_manager:
     git: https://github.com/cnf-testsuite/release_manager.git
     version: 0.1.0+git.commit.a1d7b3568d3112f737ab3ff4a7bae69a6b86970a
 
   retriable:
     git: https://github.com/sija/retriable.cr.git
-    version: 0.2.4
+    version: 0.2.5
 
   sam:
     git: https://github.com/vulk/sam.cr.git

shard.yml

@@ -49,7 +49,7 @@ dependencies:
     version: ~> 1.0.0
   kubectl_client:
     github: cnf-testsuite/kubectl_client
-    version: ~> 1.0.4
+    version: ~> 1.0.5
   cluster_tools:
     github: cnf-testsuite/cluster_tools
     version: ~> 1.0.0

spec/5g/core_spec.cr

@@ -0,0 +1,40 @@
+require "../spec_helper"
+require "colorize"
+require "../../src/tasks/utils/utils.cr"
+require "../../src/tasks/kind_setup.cr"
+require "file_utils"
+require "sam"
+
+describe "Core" do
+
+  before_all do
+    `./cnf-testsuite setup`
+    $?.success?.should be_true end
+
+  it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"]  do
+    begin
+      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
+      $?.success?.should be_true
+      response_s = `./cnf-testsuite smf_upf_heartbeat verbose`
+      Log.info {"response: #{response_s}"}
+      (/PASSED: Chaos service degradation is less than 50%/ =~ response_s).should_not be_nil
+    ensure
+      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
+      $?.success?.should be_true
+    end
+  end
+
+  it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"]  do
+    begin
+      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
+      $?.success?.should be_true
+      response_s = `./cnf-testsuite smf_upf_heartbeat verbose baseline_count=300`
+      Log.info {"response: #{response_s}"}
+      (/FAILED: Chaos service degradation is more than 50%/ =~ response_s).should_not be_nil
+    ensure
+      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
+      $?.success?.should be_true
+    end
+  end
+
+end

spec/5g/ran_spec.cr

@@ -14,14 +14,6 @@ describe "5g" do
 
   it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"]  do
     begin
-#       `/bin/bash -c "#{Dir.current}/spec/5g/key-setup.sh"`
-#       KubectlClient::Create.command("-f ./configmap.yml")
-#       Helm.fetch("openverso/open5gs --version 2.0.11 --untar")
-#       File.copy("#{Dir.current}/spec/fixtures/udm-config-open5gs.yml", "#{Dir.current}/open5gs/charts/open5gs-udm/resources/config/udm.yaml")
-#       Helm.install("open5gs #{Dir.current}/open5gs --values #{Dir.current}/spec/fixtures/5g-core-config.yml")
-#       KubectlClient::Get.wait_for_install("open5gs-pcf")
-#       $?.success?.should be_true
-# #      (/PASSED: CNF compatible with both Calico and Cilium/ =~ response_s).should_not be_nil
       Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
       $?.success?.should be_true
       response_s = `./cnf-testsuite suci_enabled verbose`

src/tasks/utils/config.cr

@@ -35,7 +35,9 @@ module CNFManager
                                      white_list_container_names: Array(String),
                                      docker_insecure_registries: Array(String) | Nil,
                                      #todo change this to an array of labels that capture all of 5g core nodes
-                                     core_label: String,
+                                     amf_label: String,
+                                     smf_label: String,
+                                     upf_label: String,
                                      ric_label: String,
                                      fiveG_core: NamedTuple(amf_pod_name: String,
                                                            mmc: String,
@@ -129,7 +131,9 @@ module CNFManager
       apn:  optional_key_as_string(config, "apn"),
       emergency:  core_emergency,
       }
-      core  = optional_key_as_string(config, "core_label")
+      core  = optional_key_as_string(config, "amf_label")
+      smf  = optional_key_as_string(config, "smf_label")
+      upf  = optional_key_as_string(config, "upf_label")
       ric = optional_key_as_string(config, "ric_label")
       if helm_directory.empty?
         working_chart_directory = "exported_chart"
@@ -208,7 +212,9 @@ module CNFManager
                                container_names: container_names,
                                white_list_container_names: white_list_container_names,
                                docker_insecure_registries: docker_insecure_registries,
-                               core_label: core,
+                               amf_label: core,
+                               smf_label: smf,
+                               upf_label: upf,
                                ric_label: ric,
                                fiveG_core: fiveG_core,
                                image_registry_fqdns: image_registry_fqdns,})

src/tasks/utils/k8s_tshark.cr

@@ -32,14 +32,49 @@ module K8sTshark
         # ClusterTools.exec_by_node_bg("tshark -ni any -a duration:120 -Y nas_5gs.mm.type_id  -T json 2>&1 | tee #{tshark_log_name}", node)
         # Log.info { "after exec by node bg" }
         # resp = tshark_log_name
-        resp = log_of_tshark_by_node(command, node, duration="120")
+        resp = log_of_tshark_by_node(command, node, duration)
       else
         resp = "label key:#{label_key} value: #{label_value} not found"
       end
       Log.info { "resp #{resp}" }
       resp
   end
 
+  def self.log_of_tshark_by_label_bg(command, label_key, label_value, duration="120") : String
+    Log.info { "log_of_tshark_by_label command label_key label value: #{command} #{label_key} #{label_value}" }
+    all_pods = KubectlClient::Get.pods_by_nodes(KubectlClient::Get.schedulable_nodes_list)
+    pods = KubectlClient::Get.pods_by_label(all_pods, label_key, label_value) 
+    first_labeled_pod = pods[0]?
+      Log.info { "first_labeled_pod: #{first_labeled_pod}" }
+    if first_labeled_pod && first_labeled_pod.dig?("metadata", "name")
+      Log.info { "first_labeled_pod #{first_labeled_pod} metadata name: #{first_labeled_pod.dig?("metadata", "name")}" }
+      pod_name = first_labeled_pod.dig("metadata", "name")
+      Log.info { "pod_name: #{pod_name}" }
+      nodes = KubectlClient::Get.nodes_by_pod(first_labeled_pod)
+      node = nodes.first
+      #create a unique name for the log
+      # rnd = Random.new
+      # name_id = rnd.next_int
+      # tshark_log_name = "/tmp/tshark-#{name_id}.json"
+      # Log.info { "tshark_log_name #{tshark_log_name}" }
+      #
+      # #tshark -ni any  -Y nas_5gs.mm.type_id -T json 2>&1 | tee hi.log 
+      # #command= -ni any  -Y nas_5gs.mm.type_id -T json
+      # #todo check if tshark running already to keep from saturating network
+      # #todo play with reducing default duration
+      # ClusterTools.exec_by_node_bg("tshark #{command} -a duration:#{duration} 2>&1 | tee #{tshark_log_name}", node)
+      # ClusterTools.exec_by_node_bg("tshark -ni any -a duration:120 -Y nas_5gs.mm.type_id  -T json 2>&1 | tee #{tshark_log_name}", node)
+      # Log.info { "after exec by node bg" }
+      # resp = tshark_log_name
+      resp = log_of_tshark_by_node_bg(command, node, duration="120")
+    else
+      resp = "label key:#{label_key} value: #{label_value} not found"
+    end
+    Log.info { "resp #{resp}" }
+    resp
+  end
+
+
   def self.log_of_tshark_by_node(command, node, duration="120") : String
     Log.info { "log_of_tshark_by_node: command #{command}" }
     #create a unique name for the log
@@ -48,6 +83,22 @@ module K8sTshark
     tshark_log_name = "/tmp/tshark-#{name_id}.json"
     Log.info { "log_of_tshark_by_node tshark_log_name #{tshark_log_name}" }
 
+    #tshark -ni any  -Y nas_5gs.mm.type_id -T json 2>&1 | tee hi.log 
+    #command= -ni any  -Y nas_5gs.mm.type_id -T json
+    #todo check if tshark running already to keep from saturating network
+    ClusterTools.exec_by_node("tshark #{command} -a duration:#{duration} 2>&1 | tee #{tshark_log_name}", node)
+    Log.info { "after exec by node bg" }
+    tshark_log_name
+  end
+
+  def self.log_of_tshark_by_node_bg(command, node, duration="120") : String
+    Log.info { "log_of_tshark_by_node: command #{command}" }
+    #create a unique name for the log
+    rnd = Random.new
+    name_id = rnd.next_int.abs
+    tshark_log_name = "/tmp/tshark-#{name_id}.json"
+    Log.info { "log_of_tshark_by_node tshark_log_name #{tshark_log_name}" }
+
     #tshark -ni any  -Y nas_5gs.mm.type_id -T json 2>&1 | tee hi.log 
     #command= -ni any  -Y nas_5gs.mm.type_id -T json
     #todo check if tshark running already to keep from saturating network
@@ -56,23 +107,45 @@ module K8sTshark
     tshark_log_name
   end
 
-  def self.regex_tshark_log(regex, tshark_log_name)
+
+  def self.regex_tshark_log_scan(regex, tshark_log_name)
+    Log.info { "regex_tshark_log regex tshark_log_name: #{regex} #{tshark_log_name}" }
+    resp = File.read("#{tshark_log_name}")
+    Log.debug { "tshark_log_name resp: #{resp}" }
+    if resp
+      Log.debug { "resp: #{resp}" }
+      ret = resp.scan(regex) 
+    else
+      Log.info { "file empty" }
+      ret = nil
+    end
+    Log.info { "#{regex}: #{ret}" }
+    ret
+  end
+
+  def self.regex_tshark_log_match(regex, tshark_log_name)
     Log.info { "regex_tshark_log regex tshark_log_name: #{regex} #{tshark_log_name}" }
-    regex_found : Bool | Nil
     resp = File.read("#{tshark_log_name}")
     Log.info { "tshark_log_name resp: #{resp}" }
     if resp
       Log.info { "resp: #{resp}" }
-      if resp =~ regex 
-        regex_found = true
-      else
-        regex_found = false
-      end
+      ret = resp =~ regex 
     else
       Log.info { "file empty" }
+      ret = nil
+    end
+    Log.info { "#{regex}: #{ret}" }
+    ret
+  end
+
+  def self.regex_tshark_log(regex, tshark_log_name)
+    Log.info { "regex_tshark_log regex tshark_log_name: #{regex} #{tshark_log_name}" }
+    regex_found : Bool | Nil
+    if regex_tshark_log_match(regex, tshark_log_name) 
+      regex_found = true
+    else
       regex_found = false
     end
-    Log.info { "#{regex}: #{regex_found}" }
     regex_found
   end
 

src/tasks/utils/oran_monitor.cr

@@ -37,7 +37,7 @@ module ORANMonitor
       resp = ClusterTools.exec_by_node("tshark --version", node)
       Log.info { "tshark must be version 4.0.3 or higher" }
       Log.info { "tshark output #{resp[:output]}" }
-      tshark_log_name = K8sTshark.log_of_tshark_by_node(command,node)
+      tshark_log_name = K8sTshark.log_of_tshark_by_node_bg(command,node)
     else
       tshark_log_name = nil
     end

src/tasks/utils/srsran.cr

@@ -14,7 +14,7 @@ module SRSRAN
 
   def self.install(config)
     Log.info {"Installing srsran"}
-    core = config.cnf_config[:core_label]? 
+    core = config.cnf_config[:amf_label]? 
     Log.info { "core: #{core}" }
     #todo use sane defaults (i.e. search for amf, upf, etc in pod names) if no 5gcore labels are present
     amf_pod_name = config.cnf_config[:fiveG_core][:amf_pod_name]? 
@@ -34,8 +34,8 @@ module SRSRAN
     emergency = config.cnf_config[:fiveG_core][:emergency]? 
     core_key : String  = ""
     core_value : String = ""
-    core_key = config.cnf_config[:core_label].split("=").first if core
-    core_value = config.cnf_config[:core_label].split("=").last if core
+    core_key = config.cnf_config[:amf_label].split("=").first if core
+    core_value = config.cnf_config[:amf_label].split("=").last if core
     if core 
       all_pods = KubectlClient::Get.pods_by_nodes(KubectlClient::Get.schedulable_nodes_list)
       ueran_pods = KubectlClient::Get.pods_by_label(all_pods, "app.kubernetes.io/name", "ueransim-gnb")

src/tasks/utils/task.cr

@@ -84,7 +84,9 @@ module CNFManager
                                             container_names: [{"name" =>  "", "rolling_update_test_tag" => ""}],
                                             white_list_container_names: [""],
                                             docker_insecure_registries: [] of String,
-                                            core_label: "",
+                                            amf_label: "",
+                                            smf_label: "",
+                                            upf_label: "",
                                             ric_label: "",
                                             fiveG_core: {amf_pod_name:  "",
                                                            mmc:  "",

src/tasks/utils/ueransim.cr

@@ -20,7 +20,7 @@ module UERANSIM
 
   def self.install(config)
     Log.info {"Installing ueransim with 5g config"}
-    core = config.cnf_config[:core_label]? 
+    core = config.cnf_config[:amf_label]? 
     Log.info { "core: #{core}" }
     #todo use sane defaults (i.e. search for amf, upf, etc in pod names) if no 5gcore labels are present
     amf_pod_name = config.cnf_config[:fiveG_core][:amf_pod_name]? 
@@ -40,8 +40,8 @@ module UERANSIM
     emergency = config.cnf_config[:fiveG_core][:emergency]? 
     core_key : String  = ""
     core_value : String = ""
-    core_key = config.cnf_config[:core_label].split("=").first if core
-    core_value = config.cnf_config[:core_label].split("=").last if core
+    core_key = config.cnf_config[:amf_label].split("=").first if core
+    core_value = config.cnf_config[:amf_label].split("=").last if core
     if core 
       all_pods = KubectlClient::Get.pods_by_nodes(KubectlClient::Get.schedulable_nodes_list)
       ueran_pods = KubectlClient::Get.pods_by_label(all_pods, "app.kubernetes.io/name", "ueransim-gnb")