Skip to content

Commit

Permalink
Build the nightly pipeline in parallel
Browse files Browse the repository at this point in the history 
Signed-off-by: Joonas Rautiola <[email protected]>
  • Loading branch information
@joinemm
joinemm committed Nov 6, 2024
1 parent 97e20dc commit b886fb0
Show file tree
Hide file tree
Showing 2 changed files with 220 additions and 92 deletions.
294 changes: 202 additions & 92 deletions ghaf-nightly-pipeline.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,78 @@ properties([
githubProjectProperty(displayName: '', projectUrlStr: REPO_URL),
])

def targets = [
// docs
[ system: "x86_64-linux", target: "doc",
archive: false
],
[ system: "aarch64-linux", target: "doc",
archive: false
],

// lenovo x1
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug",
archive: true, scs: true, hwtest_device: "lenovo-x1"
],
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug-installer",
archive: true, scs: true
],
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-release",
archive: true, scs: true
],
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-release-installer",
archive: true, scs: true
],

// nvidia orin
[ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug",
archive: true, scs: true, hwtest_device: "orin-agx"
],
[ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug",
archive: true, scs: true, hwtest_device: "orin-nx"
],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64",
archive: true, scs: true, hwtest_device: "orin-agx"
],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64",
archive: true, scs: true, hwtest_device: "orin-nx"
],

// others
[ system: "x86_64-linux", target: "generic-x86_64-debug",
archive: true, hwtest_device: "nuc"
],
[ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64",
archive: true, scs: true, hwtest_device: "riscv"
],
[ system: "x86_64-linux", target: "nxp-imx8mp-evk-debug",
archive: true, scs: true
],
]

hydrajobs_targets = [
// nvidia orin with bpmp enabled
[ system: "aarch64-linux",target: "nvidia-jetson-orin-agx-debug-bpmp",
archive: true
],
[ system: "aarch64-linux",target: "nvidia-jetson-orin-nx-debug-bpmp",
archive: true
],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-bpmp-from-x86_64",
archive: true
],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-bpmp-from-x86_64",
archive: true
],
]

target_jobs = [:]

////////////////////////////////////////////////////////////////////////////////

pipeline {
agent { label 'built-in' }

triggers {
// We could use something like cron('@midnight') here, but since we
// archive the images, this pipeline would then generate many
Expand All @@ -27,12 +95,15 @@ pipeline {
// we trigger based one-time daily poll at 23:00 instead:
pollSCM('0 23 * * *')
}

options {
disableConcurrentBuilds()
timestamps ()
buildDiscarder(logRotator(numToKeepStr: '100'))
}

stages {

stage('Checkout') {
steps {
script { utils = load "utils.groovy" }
Expand All @@ -50,106 +121,145 @@ pipeline {
}
}
}
stage('Build x86_64') {
steps {
dir(WORKDIR) {
script {
utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'archive')
utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug-installer', 'archive')
utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release', 'archive')
utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release-installer', 'archive')
utils.nix_build('.#packages.x86_64-linux.generic-x86_64-debug', 'archive')
utils.nix_build('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'archive')
utils.nix_build('.#hydraJobs.nvidia-jetson-orin-agx-debug-bpmp-from-x86_64.x86_64-linux', 'archive')
utils.nix_build('.#hydraJobs.nvidia-jetson-orin-nx-debug-bpmp-from-x86_64.x86_64-linux', 'archive')
utils.nix_build('.#packages.x86_64-linux.doc')
}
}
}
}
stage('Build aarch64') {
steps {
dir(WORKDIR) {
script {
utils.nix_build('.#packages.aarch64-linux.nxp-imx8mp-evk-debug', 'archive')
utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'archive')
utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'archive')
utils.nix_build('.#hydraJobs.nvidia-jetson-orin-agx-debug-bpmp.aarch64-linux', 'archive')
utils.nix_build('.#hydraJobs.nvidia-jetson-orin-nx-debug-bpmp.aarch64-linux', 'archive')
utils.nix_build('.#packages.aarch64-linux.doc')
}
}
}
}
stage('Provenance') {
steps {
dir(WORKDIR) {
script {
utils.sbomnix('provenance', '.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64')
utils.sbomnix('provenance', '.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64')
utils.sbomnix('provenance', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug')
utils.sbomnix('provenance', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug-installer')
utils.sbomnix('provenance', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release')
utils.sbomnix('provenance', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release-installer')
utils.sbomnix('provenance', '.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64')
utils.sbomnix('provenance', '.#packages.aarch64-linux.nxp-imx8mp-evk-debug')
utils.sbomnix('provenance', '.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug')
utils.sbomnix('provenance', '.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug')
}
}
}
}
stage('SBOM') {
steps {
dir(WORKDIR) {
script {
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64')
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64')
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug')
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug-installer')
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release')
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release-installer')
utils.sbomnix('sbomnix', '.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64')
utils.sbomnix('sbomnix', '.#packages.aarch64-linux.nxp-imx8mp-evk-debug')
utils.sbomnix('sbomnix', '.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug')
utils.sbomnix('sbomnix', '.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug')
}
}
}
}
stage('Vulnxscan') {

stage('Evaluate') {
steps {
dir(WORKDIR) {
script {
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64')
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64')
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug')
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug-installer')
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release')
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.lenovo-x1-carbon-gen11-release-installer')
utils.sbomnix('vulnxscan', '.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64')
utils.sbomnix('vulnxscan', '.#packages.aarch64-linux.nxp-imx8mp-evk-debug')
utils.sbomnix('vulnxscan', '.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug')
utils.sbomnix('vulnxscan', '.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug')
utils.nix_eval_hydrajobs(hydrajobs_targets)
return
// targets = utils.nix_eval_jobs(targets, "packages")
// targets = targets + utils.nix_eval_jobs(hydrajobs_targets, "hydraJobs")

targets.each {
def timestampBegin = ""
def timestampEnd = ""
def scsdir = "scs/${target}/scs"
def target = it.system + "." + it.target

target_jobs["${it.target} (${it.system})"] = {
stage("Build ${target}") {
def opts = ""
if (it.archive) {
opts = "--out-link archive/${target}"
} else {
opts = "--no-link"
}
try {
if (drvPath) {
timestampBegin = sh(script: "date +%s", returnStdout: true).trim()
sh "nix build -L ${drvPath}\\^* ${opts}"
timestampEnd = sh(script: "date +%s", returnStdout: true).trim()

// only attempt signing if there is something to sign
if (it.archive) {
def img_relpath = find_img_relpath(target, "archive")
sign_file("archive/${img_relpath}", "sig/${img_relpath}.sig", "INT-Ghaf-Devenv-Image")
};
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
}
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
unstable("FAILED: ${target}")
currentBuild.result = "FAILURE"
println "Error: ${e.toString()}"
}
}

if (scs) {
stage("Provenance (${target})") {
def externalParams = """
{
"target": {
"name": "#packages.${target}",
"repository": "${env.TARGET_REPO}",
"ref": "${env.TARGET_COMMIT}"
},
"workflow": {
"name": "${env.JOB_NAME}",
"repository": "${env.GIT_URL}",
"ref": "${env.GIT_COMMIT}"
},
"job": "${env.JOB_NAME}",
"jobParams": ${JsonOutput.toJson(params)},
"buildRun": "${env.BUILD_ID}"
}
"""
// this environment block is only valid for the scope of this stage,
// preventing timestamp collision when provenances are built in parallel
withEnv([
'PROVENANCE_BUILD_TYPE="https://github.com/tiiuae/ghaf-infra/blob/ea938e90/slsa/v1.0/L1/buildtype.md"',
"PROVENANCE_BUILDER_ID=${env.JENKINS_URL}",
"PROVENANCE_INVOCATION_ID=${env.BUILD_URL}",
"PROVENANCE_TIMESTAMP_BEGIN=${timestampBegin}",
"PROVENANCE_TIMESTAMP_FINISHED=${timestampEnd}",
"PROVENANCE_EXTERNAL_PARAMS=${externalParams}"
]) {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
def outpath = "${scsdir}/provenance.json"
sh """
mkdir -p ${scsdir}
sh "provenance ${drvPath} --recursive --out ${outpath}
"""
sign_file(outpath, "sig/${outpath}.sig", "INT-Ghaf-Devenv-Provenance")
}
}
}

stage("SBOM (${target})") {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh """
mkdir -p ${scsdir}
cd ${scsdir}
sbomnix ${drvPath}
cd ..
"""
}
}

stage("Vulnxscan (${target})") {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh """
mkdir -p ${scsdir}
vulnxscan ${drvPath} --out vulns.csv
csvcut vulns.csv --not-columns sortcol | csvlook -I >${scsdir}/vulns.txt
"""
}
}
}

if (it.archive) {
stage("Archive ${target}") {
script {
archive_artifacts("archive", target)
archive_artifacts("sig", target)
if (it.scs) {
archive_artifacts("scs", target)
}
}
}
}

if (it.hwtest_device != null) {
stage("Test ${target}") {
script {
ghaf_parallel_hw_test(target, it.hwtest_device, '_boot_bat_perf_')
}
}
}
}
}
}
}
}
}
stage('HW test') {

stage('Build targets') {
steps {
dir(WORKDIR) {
script {
testset = "_boot_bat_perf_"
utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'orin-agx', testset)
utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'orin-agx', testset)
utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'orin-nx', testset)
utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'orin-nx', testset)
utils.ghaf_hw_test('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'lenovo-x1', testset)
utils.ghaf_hw_test('.#packages.x86_64-linux.generic-x86_64-debug', 'nuc', testset)
utils.ghaf_hw_test('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'riscv', testset)
}
script {
parallel target_jobs
}
}
}
Expand Down
18 changes: 18 additions & 0 deletions utils.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,20 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset=
archive_artifacts("ghaf-parallel-hw-test", flakeref_trimmed)
}

def nix_eval_hydrajobs(List<Map> targets) {
def targetList = "\"${targets.join('" "')}\""
def jsonSlurper = new JsonSlurper()
def object = jsonSlurper.parseText(sh(script: """
nix-eval-jobs --gc-roots-dir gcroots --force-recurse --expr ' \
let \
flake = builtins.getFlake ("git+file://" + toString ./.); \
lib = (import flake.inputs.nixpkgs { }).lib; \
in lib.getAttrs [ ${targetsList} ] flake.hydraJobs;'
""", returnStdout: true).trim())

println "${object}"
}

def nix_eval_jobs(List<Map> targets) {
// transform target names into valid nix arrays to be plugged into the expression below
def x86_targets = "\"${targets.findAll { it.system == "x86_64-linux" }.target.join('" "')}\""
Expand All @@ -325,6 +339,10 @@ def nix_eval_jobs(List<Map> targets) {

// target's name and derivation path are read from jobs.json and written into into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()


}

return this
Expand Down

0 comments on commit b886fb0

Please sign in to comment.