From baaffc66b2512951782eb74bda15d3c010b07c84 Mon Sep 17 00:00:00 2001 From: Joonas Rautiola Date: Wed, 9 Oct 2024 10:39:31 +0300 Subject: [PATCH 1/2] Build the main pipeline in parallel --- ghaf-main-pipeline.groovy | 111 +++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/ghaf-main-pipeline.groovy b/ghaf-main-pipeline.groovy index 24c47e4..fe99e78 100644 --- a/ghaf-main-pipeline.groovy +++ b/ghaf-main-pipeline.groovy @@ -15,7 +15,32 @@ properties([ githubProjectProperty(displayName: '', projectUrlStr: REPO_URL), ]) -//////////////////////////////////////////////////////////////////////////////// +// Which attribute of the flake to evaluate for building +def flakeAttr = ".#hydraJobs" + +// Target names must be direct children of the above +def targets = [ + [ target: "docs.aarch64-linux", + hwtest_device: null ], + [ target: "docs.x86_64-linux", + hwtest_device: null ], + [ target: "generic-x86_64-debug.x86_64-linux", + hwtest_device: "nuc" ], + [ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux", + hwtest_device: "lenovo-x1" ], + [ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux", + hwtest_device: null ], + [ target: "nvidia-jetson-orin-agx-debug.aarch64-linux", + hwtest_device: "orin-agx" ], + [ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux", + hwtest_device: "orin-agx" ], + [ target: "nvidia-jetson-orin-nx-debug.aarch64-linux", + hwtest_device: "orin-nx" ], + [ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux", + hwtest_device: "orin-nx" ], +] + +target_jobs = [:] pipeline { agent { label 'built-in' } @@ -23,7 +48,6 @@ pipeline { pollSCM('* * * * *') } options { - disableConcurrentBuilds() timestamps () buildDiscarder(logRotator(numToKeepStr: '100')) } @@ -44,43 +68,68 @@ pipeline { } } } - stage('Build x86_64') { - steps { - dir(WORKDIR) { - script { - utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'archive') - utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'archive') - utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'archive') - utils.nix_build('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'archive') - utils.nix_build('.#packages.x86_64-linux.generic-x86_64-debug', 'archive') - utils.nix_build('.#packages.x86_64-linux.doc') - } - } - } - } - stage('Build aarch64') { + + stage('Evaluate') { steps { dir(WORKDIR) { script { - utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'archive') - utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'archive') - utils.nix_build('.#packages.aarch64-linux.doc') + // nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json + sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json" + + // jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt + sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt" + + targets.each { + def target = it['target'] + + // row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path + def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim() + + target_jobs[target] = { + stage("Build ${target}") { + def opts = "" + if (it['hwtest_device'] != null) { + opts = "--out-link archive/${target}" + } else { + opts = "--no-link" + } + try { + if (drvPath) { + sh "nix build -L ${drvPath}\\^* ${opts}" + } else { + error("Target \"${target}\" was not found in ${flakeAttr}") + } + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + unstable("FAILED: ${target}") + currentBuild.result = "FAILURE" + println "Error: ${e.toString()}" + } + } + + if (it['hwtest_device'] != null) { + stage("Archive ${target}") { + script { + utils.archive_artifacts("archive", target) + } + } + + stage("Test ${target}") { + utils.ghaf_parallel_hw_test(target, it['hwtest_device']) + } + } + } + } } } } } - stage('HW test') { + + stage('Build targets') { steps { - dir(WORKDIR) { - script { - utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'orin-agx') - utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'orin-agx') - utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'orin-nx') - utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'orin-nx') - utils.ghaf_hw_test('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'lenovo-x1') - utils.ghaf_hw_test('.#packages.x86_64-linux.generic-x86_64-debug', 'nuc') - utils.ghaf_hw_test('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'riscv') - } + script { + parallel target_jobs } } } From 0fcebfe721bf3d83f91ec73f305186a58db9dcaa Mon Sep 17 00:00:00 2001 From: Joonas Rautiola Date: Wed, 16 Oct 2024 11:13:42 +0300 Subject: [PATCH 2/2] Remove testagent_ prefix and use device names as labels Signed-off-by: Joonas Rautiola --- utils.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils.groovy b/utils.groovy index 7c94202..85bec55 100644 --- a/utils.groovy +++ b/utils.groovy @@ -187,7 +187,7 @@ def sign_file(String path, String sigfile, String cert="INT-Ghaf-Devenv-Common") } def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_') { - testagent_nodes = nodesByLabel(label: 'testagent', offline: false) + testagent_nodes = nodesByLabel(label: "$device_config", offline: false) if (!testagent_nodes) { println "Warning: Skipping HW test '$flakeref', no test agents online" unstable("No test agents online") @@ -217,7 +217,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_') job: "ghaf-hw-test", propagate: false, parameters: [ - string(name: "LABEL", value: "testagent"), + string(name: "LABEL", value: "$device_config"), string(name: "DEVICE_CONFIG_NAME", value: "$device_config"), string(name: "IMG_URL", value: "$img_url"), string(name: "DESC", value: "$description"), @@ -247,7 +247,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_') } def ghaf_parallel_hw_test(String flakeref, String device_config, String testset='_boot_') { - testagent_nodes = nodesByLabel(label: "testagent_$device_config", offline: false) + testagent_nodes = nodesByLabel(label: "$device_config", offline: false) if (!testagent_nodes) { println "Warning: Skipping HW test '$flakeref', no test agents online" unstable("No test agents online") @@ -275,7 +275,7 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset= job: "ghaf-parallel-hw-test", propagate: false, parameters: [ - string(name: "LABEL", value: "testagent_$device_config"), + string(name: "LABEL", value: "$device_config"), string(name: "DEVICE_CONFIG_NAME", value: "$device_config"), string(name: "IMG_URL", value: "$img_url"), string(name: "DESC", value: "$description"),