From f9b14e735fe695c6cc27507035f3bdda0249bb4a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 24 Jan 2025 10:45:57 -0800 Subject: [PATCH 1/3] allow the sparse matrix to generate fully --- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 8e18e2479..a30ba45b8 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -44,6 +44,7 @@ param ( [Parameter(Mandatory = $False)][array] $Filters, [Parameter(Mandatory = $False)][array] $IndirectFilters, [Parameter(Mandatory = $False)][array] $Replace, + [Parameter(Mandatory = $False)][bool] $SparseIndirect = $true, [Parameter(Mandatory = $False)][int] $PackagesPerPRJob = 10, [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) ) @@ -72,7 +73,8 @@ function QueuePop([ref]$queue) { function GeneratePRMatrixForBatch { param ( - [Parameter(Mandatory = $true)][array] $Packages + [Parameter(Mandatory = $true)][array] $Packages, + [Parameter(Mandatory = $true)][bool] $FullSparseMatrix ) $OverallResult = @() @@ -85,6 +87,8 @@ function GeneratePRMatrixForBatch { $directBatch = $Packages[0].IncludedForValidation -eq $false Write-Host "Generating matrix for $($directBatch ? 'direct' : 'indirect') packages" + $batchNamePrefix = $($directBatch ? 'b' : 'ib') + # The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig # to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. $matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs" @@ -134,10 +138,10 @@ function GeneratePRMatrixForBatch { # we only need to modify the generated job name if there is more than one matrix config + batch $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 - # if we are doing direct packages, we need to walk the batches and duplicate the matrix config for each batch, fully assigning + # if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages - if ($directBatch) { - $batchSuffixNecessary = $packageBatches.Length -gt 1 + if ($directBatch -or $FullSparseMatrix) { + $batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0) $batchCounter = 1 foreach ($batch in $packageBatches) { @@ -155,7 +159,7 @@ function GeneratePRMatrixForBatch { } if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_b$batchCounter" + $outputItem["name"] = $outputItem["name"] + "$batchPrefix$batchCounter" } $OverallResult += $outputItem @@ -236,7 +240,7 @@ if ($indirectPackages) { foreach($artifact in $indirectPackages) { Write-Host "-> $($artifact.ArtifactName)" } - $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages + $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages -FullSparseMatrix !$SparseIndirect } $serialized = SerializePipelineMatrix $OverallResult From 4ed1cd8aa54bfa64ecfd94d7b6bb71ddde16e724 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 24 Jan 2025 13:02:02 -0800 Subject: [PATCH 2/3] allow indirect packages to run the full matrix, instead of being forced to sparse --- eng/common/pipelines/templates/jobs/generate-job-matrix.yml | 6 +++++- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index d1db47b30..7cb3a785c 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -54,6 +54,9 @@ parameters: - name: PRMatrixIndirectFilters type: object default: [] +- name: PRMatrixSparseIndirect + type: boolean + default: true # Mappings to OS name required at template compile time by 1es pipeline templates - name: Pools type: object @@ -142,7 +145,8 @@ jobs: -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` -IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' ` -Replace '${{ join(''',''', parameters.MatrixReplace) }}' ` - -PackagesPerPRJob ${{ parameters.PRJobBatchSize }} + -PackagesPerPRJob ${{ parameters.PRJobBatchSize }} ` + -SparseIndirect $${{ parameters.PRMatrixSparseIndirect }} displayName: Create ${{ pool.name }} PR Matrix name: vm_job_matrix_pr_${{ pool.name }} diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index a30ba45b8..23bb513da 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -74,7 +74,7 @@ function QueuePop([ref]$queue) { function GeneratePRMatrixForBatch { param ( [Parameter(Mandatory = $true)][array] $Packages, - [Parameter(Mandatory = $true)][bool] $FullSparseMatrix + [Parameter(Mandatory = $false)][bool] $FullSparseMatrix = $false ) $OverallResult = @() @@ -159,7 +159,7 @@ function GeneratePRMatrixForBatch { } if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "$batchPrefix$batchCounter" + $outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter" } $OverallResult += $outputItem @@ -184,7 +184,7 @@ function GeneratePRMatrixForBatch { } if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_ib$batchCounter" + $outputItem["name"] = $outputItem["name"] + "_$batchNamePrefix$batchCounter" } # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results # we will add the cloned version to OverallResult From 77423560f3b49290837d75ae4a60b15d662692be Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 27 Jan 2025 14:22:03 -0800 Subject: [PATCH 3/3] need this boolean inversion to not it --- eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 23bb513da..805b2d934 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -240,7 +240,7 @@ if ($indirectPackages) { foreach($artifact in $indirectPackages) { Write-Host "-> $($artifact.ArtifactName)" } - $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages -FullSparseMatrix !$SparseIndirect + $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages -FullSparseMatrix (-not $SparseIndirect) } $serialized = SerializePipelineMatrix $OverallResult