diff --git a/ghaf-test-boot.groovy b/ghaf-hw-test.groovy similarity index 59% rename from ghaf-test-boot.groovy rename to ghaf-hw-test.groovy index ac4593b..648f0db 100644 --- a/ghaf-test-boot.groovy +++ b/ghaf-hw-test.groovy @@ -17,13 +17,70 @@ def run_cmd(String cmd) { return sh(script: cmd, returnStdout:true).trim() } -def get_test_conf_property(file_path, device, property) { - // get wanted property data from wanted device from test_config.json file +def get_test_conf_property(String file_path, String device, String property) { + // Get the requested device property data from test_config.json file def device_data = readJSON file: file_path property_data = "${device_data['addresses'][device][property]}" + println "Got device '${device}' property '${property}' value: '${property_data}'" return property_data } +def ghaf_robot_test(String testname='boot') { + if (!env.DEVICE_TAG) { + sh "echo 'DEVICE_TAG not set'; exit 1" + } + if (!env.DEVICE_NAME) { + sh "echo 'DEVICE_NAME not set'; exit 1" + } + // TODO: do we really need credentials to access the target devices? + // Target devices are connected to the testagent, which itself is + // only available over a private network. What is the risk + // we are protecting against by having additional authentication + // for the test devices? + // The current configuration requires additional manual configuration + // on the jenkins UI to add the following secrets: + withCredentials([ + string(credentialsId: 'testagent-dut-pass', variable: 'DUT_PASS'), + string(credentialsId: 'testagent-plug-pass', variable: 'PLUG_PASS'), + string(credentialsId: 'testagent-switch-token', variable: 'SW_TOKEN'), + string(credentialsId: 'testagent-switch-secret', variable: 'SW_SECRET'), + ]) { + dir('Robot-Framework/test-suites') { + env.INCLUDE_TEST_TAGS = "${testname}AND${env.DEVICE_TAG}" + sh 'rm -f *.png output.xml report.html log.html' + // On failure, continue the pipeline execution + catchError(stageResult: 'FAILURE', buildResult: 'FAILURE') { + // Pass the secrets to the shell as environment variables, as we + // don't want Groovy to interpolate them. Similary, we pass + // other variables as environment variables to shell. + // Ref: https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#string-interpolation + sh ''' + nix run .#ghaf-robot -- \ + -v DEVICE:$DEVICE_NAME \ + -v LOGIN:ghaf \ + -v PASSWORD:$DUT_PASS \ + -v PLUG_USERNAME:ville-pekka.juntunen@unikie.com \ + -v PLUG_PASSWORD:$PLUG_PASS \ + -v SWITCH_TOKEN:$SW_TOKEN \ + -v SWITCH_SECRET:$SW_SECRET \ + -v BUILD_ID:${BUILD_NUMBER} \ + -i $INCLUDE_TEST_TAGS . + ''' + // Move the test output (if any) to a subdirectory + sh """ + rm -fr $testname; mkdir -p $testname + mv -f *.png output.xml report.html log.html $testname/ || true + """ + if (testname == 'boot') { + // Set an environment variable to indicate boot test passed + env.BOOT_PASSED = 'true' + } + } + } + } +} + + //////////////////////////////////////////////////////////////////////////////// pipeline { @@ -84,19 +141,19 @@ pipeline { sh "exit 1" } if(["orin-agx"].contains(params.DEVICE_CONFIG_NAME)) { - env.DEVICE = 'OrinAGX1' - env.INCLUDE_TEST_TAGS = 'bootANDorin-agx' + env.DEVICE_NAME = 'OrinAGX1' + env.DEVICE_TAG = 'orin-agx' } else if(["orin-nx"].contains(params.DEVICE_CONFIG_NAME)) { - env.DEVICE = 'OrinNX1' - env.INCLUDE_TEST_TAGS = 'bootANDorin-nx' + env.DEVICE_NAME = 'OrinNX1' + env.DEVICE_TAG = 'orin-nx' } else if(["lenovo-x1"].contains(params.DEVICE_CONFIG_NAME)) { - env.DEVICE = 'LenovoX1-2' - env.INCLUDE_TEST_TAGS = 'bootANDlenovo-x1' + env.DEVICE_NAME = 'LenovoX1-2' + env.DEVICE_TAG = 'lenovo-x1' } else { println "Error: unsupported device config '${params.DEVICE_CONFIG_NAME}'" sh "exit 1" } - hub_serial = get_test_conf_property(CONF_FILE_PATH, env.DEVICE, 'usbhub_serial') + hub_serial = get_test_conf_property(CONF_FILE_PATH, env.DEVICE_NAME, 'usbhub_serial') mount_cmd = "/run/wrappers/bin/sudo AcronameHubCLI -u 0 -s ${hub_serial}; sleep 10" unmount_cmd = "/run/wrappers/bin/sudo AcronameHubCLI -u 1 -s ${hub_serial}" // Mount the target disk @@ -129,52 +186,40 @@ pipeline { stage('Boot test') { steps { script { - // TODO: do we really need credentials to access the target devices? - // Target devices are connected to the testagent, which itself is - // only available over a private network. What is the risk - // we are protecting against by having additional authentication - // for the test devices? - // The current configuration requires additional manual configuration - // on the jenkins UI to add the following secrets: - withCredentials([ - string(credentialsId: 'testagent-dut-pass', variable: 'DUT_PASS'), - string(credentialsId: 'testagent-plug-pass', variable: 'PLUG_PASS'), - string(credentialsId: 'testagent-switch-token', variable: 'SW_TOKEN'), - string(credentialsId: 'testagent-switch-secret', variable: 'SW_SECRET'), - ]) { - dir('Robot-Framework/test-suites') { - sh 'rm -f *.png output.xml report.html log.html' - // Pass the secrets to the shell as environment variables, as we - // don't want Groovy to interpolate them. Similary, we pass - // other variables as environment variables to shell. - // Ref: https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#string-interpolation - sh ''' - nix run .#ghaf-robot -- \ - -v DEVICE:$DEVICE \ - -v LOGIN:ghaf \ - -v PASSWORD:$DUT_PASS \ - -v PLUG_USERNAME:ville-pekka.juntunen@unikie.com \ - -v PLUG_PASSWORD:$PLUG_PASS \ - -v SWITCH_TOKEN:$SW_TOKEN \ - -v SWITCH_SECRET:$SW_SECRET \ - -i $INCLUDE_TEST_TAGS . - ''' - } - } + env.BOOT_PASSED = 'false' + ghaf_robot_test('boot') + println "Boot test passed: ${env.BOOT_PASSED}" + } + } + } + stage('Bat test') { + when { expression { env.BOOT_PASSED == 'true' } } + steps { + script { + ghaf_robot_test('bat') + } + } + } + stage('Perf test') { + when { expression { env.BOOT_PASSED == 'true' } } + steps { + script { + ghaf_robot_test('performance') } } } } post { always { + // Publish all results under Robot-Framework/test-suites subfolders step( [$class: 'RobotPublisher', archiveDirName: 'robot-plugin', outputPath: 'Robot-Framework/test-suites', - outputFileName: 'output.xml', + outputFileName: '**/output.xml', disableArchiveOutput: false, - reportFileName: 'report.html', - logFileName: 'log.html', + reportFileName: '**/report.html', + logFileName: '**/log.html', passThreshold: 0, unstableThreshold: 0, onlyCritical: true, diff --git a/ghaf-main-pipeline.groovy b/ghaf-main-pipeline.groovy index 96e8350..854c5f8 100644 --- a/ghaf-main-pipeline.groovy +++ b/ghaf-main-pipeline.groovy @@ -75,11 +75,11 @@ pipeline { dir(WORKDIR) { script { jenkins_url = "https://ghaf-jenkins-controller-dev.northeurope.cloudapp.azure.com" - utils.boot_test('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'orin-agx', jenkins_url) - utils.boot_test('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'orin-agx', jenkins_url) - utils.boot_test('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'orin-nx', jenkins_url) - utils.boot_test('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'orin-nx', jenkins_url) - utils.boot_test('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'lenovo-x1', jenkins_url) + utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'orin-agx', jenkins_url) + utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'orin-agx', jenkins_url) + utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'orin-nx', jenkins_url) + utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'orin-nx', jenkins_url) + utils.ghaf_hw_test('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'lenovo-x1', jenkins_url) } } } diff --git a/utils.groovy b/utils.groovy index bf3585c..ab21134 100644 --- a/utils.groovy +++ b/utils.groovy @@ -147,7 +147,10 @@ def find_img_relpath(String flakeref, String subdir) { return img_relpath } -def boot_test(String flakeref, String device_config, String jenkins_url, String subdir='archive') { +def ghaf_hw_test(String flakeref, String device_config, String jenkins_url, String subdir='archive') { + // TODO: we might want to allow running only a subset of HW tests. + // Right now, calling this function will always invoke all the + // robot framework tests: boot, bat, and performance. testagent_nodes = nodesByLabel(label: 'testagent', offline: false) if (!testagent_nodes) { println "Warning: Skipping boot test '$flakeref', no test agents online" @@ -167,10 +170,10 @@ def boot_test(String flakeref, String device_config, String jenkins_url, String // 'short' flakeref: everything after the last occurence of '.' (if any) flakeref_short = flakeref_trim(flakeref).replaceAll(/.*\.+/,"") description = "Triggered by ${build_href}
(${flakeref_short})" - // Trigger a build in 'ghaf-test-boot' pipeline. + // Trigger a build in 'ghaf-hw-test' pipeline. // 'build' step is documented in https://plugins.jenkins.io/pipeline-build-step/ build( - job: "ghaf-test-boot", + job: "ghaf-hw-test", propagate: true, parameters: [ string(name: "LABEL", value: "testagent"),