diff --git a/build/ci-build.yml b/build/ci-build.yml index ac0c22b5..d36e3b0c 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -18,6 +18,10 @@ parameters: - name: 'Package.Version.ManualTrigger' type: string default: 'preview' + - name: azureServiceConnection + displayName: 'Azure service connection' + type: string + default: 'Azure Codit-Arcus Service Principal' resources: repositories: @@ -27,7 +31,6 @@ resources: endpoint: arcus-azure variables: - - group: 'Arcus Observability - Integration Testing' - group: 'Arcus - GitHub Package Registry' - group: 'Build Configuration' - template: ./variables/build.yml @@ -75,78 +78,27 @@ stages: dependsOn: Build condition: succeeded() jobs: - - job: UnitTests - displayName: 'Run unit tests' - pool: - vmImage: '$(Vm.Image)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download build artifacts' - inputs: - artifact: 'Build' - path: '$(Build.SourcesDirectory)' - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.PreviousVersion)' - - template: test/run-unit-tests.yml@templates - parameters: - dotnetSdkVersion: '$(DotNet.Sdk.Version)' - includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) - projectName: '$(Project).Tests.Unit' - + - template: templates/run-unit-tests.yml + - stage: IntegrationTests displayName: Integration Tests dependsOn: Build condition: succeeded() jobs: - - job: IntegrationTests - displayName: 'Run integration tests' - pool: - vmImage: '$(Vm.Image)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download build artifacts' - inputs: - artifact: 'Build' - path: '$(Build.SourcesDirectory)' - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.PreviousVersion)' - - template: test/run-integration-tests.yml@templates - parameters: - dotnetSdkVersion: '$(DotNet.Sdk.Version)' - includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) - projectName: '$(Project).Tests.Integration' - category: 'Integration' + - template: templates/run-self-contained-integration-tests.yml + parameters: + azureServiceConnection: '${{ parameters.azureServiceConnection }}' - stage: DockerTests displayName: Docker Tests dependsOn: Build condition: succeeded() jobs: - - job: DockerTests - displayName: 'Run Docker tests' - pool: - vmImage: '$(Vm.Image)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download build artifacts' - inputs: - artifact: 'Build' - path: '$(Build.SourcesDirectory)' - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.PreviousVersion)' - - template: templates/run-docker-integration-tests.yml - parameters: - dockerProjectName: '$(Project).Tests.Runtimes.AzureFunction' - httpPort: '$(AzureFunctions.HttpPort)' + - template: templates/run-docker-integration-tests.yml + parameters: + dockerProjectName: '$(Project).Tests.Runtimes.AzureFunction' + httpPort: '$(AzureFunctions.HttpPort)' + azureServiceConnection: '${{ parameters.azureServiceConnection }}' - stage: ReleaseToMyget displayName: 'Release to MyGet' diff --git a/build/deploy-test-resources.yml b/build/deploy-test-resources.yml new file mode 100644 index 00000000..43642e29 --- /dev/null +++ b/build/deploy-test-resources.yml @@ -0,0 +1,75 @@ +name: Arcus Observability - Deploy test resources + +trigger: none +pr: none + +parameters: + - name: azureServiceConnection + displayName: 'Azure service connection' + type: string + default: 'Azure Codit-Arcus Service Principal' + - name: resourceGroupName + displayName: 'Resource group name' + default: arcus-observability-dev-we-rg + +variables: + - template: ./variables/build.yml + - template: ./variables/test.yml + +stages: + - stage: Deploy + jobs: + - job: DeployBicep + displayName: 'Deploy test resources' + pool: + vmImage: '$(Vm.Image)' + steps: + - task: AzureCLI@2 + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + azureSubscription: '${{ parameters.azureServiceConnection }}' + addSpnToEnvironment: true + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module -Name Arcus.Scripting.DevOps -AllowClobber + az extension add --name application-insights + + $objectId = (az ad sp show --id $env:servicePrincipalId | ConvertFrom-Json).id + az deployment sub create ` + --location westeurope ` + --template-file ./build/templates/deploy-test-resources.bicep ` + --parameters location=westeurope ` + --parameters resourceGroupName=${{ parameters.resourceGroupName }} ` + --parameters appInsightsName=${{ variables['ApplicationInsights.Name'] }} ` + --parameters instrumentationKey_secretName=${{ variables['ApplicationInsights.InstrumentationKey.SecretName'] }} ` + --parameters applicationId_secretName=${{ variables['ApplicationInsights.ApplicationId.SecretName'] }} ` + --parameters keyVaultName=${{ variables['KeyVault.Name'] }} ` + --parameters servicePrincipal_objectId=$objectId ` + | ConvertFrom-Json + + $apiKeyName = 'Arcus Observability Integration Tests API Key' + if (az monitor app-insights api-key show ` + --app ${{ variables['ApplicationInsights.Name'] }} ` + --resource-group ${{ parameters.resourceGroupName }} ` + --api-key $apiKeyName) { + az monitor app-insights api-key delete ` + --api-key $apiKeyName ` + --app ${{ variables['ApplicationInsights.Name'] }} ` + --resource-group ${{ parameters.resourceGroupName }} ` + --yes + } + + $apiKeyOutput = az monitor app-insights api-key create ` + --api-key $apiKeyName ` + --app ${{ variables['ApplicationInsights.Name'] }} ` + --resource-group ${{ parameters.resourceGroupName }} ` + --read-properties ReadTelemetry ` + | ConvertFrom-Json + + az keyvault secret set ` + --vault-name ${{ variables['KeyVault.Name'] }} ` + --name ${{ variables['ApplicationInsights.ApiKey.SecretName'] }} ` + --value $apiKeyOutput.apiKey \ No newline at end of file diff --git a/build/nuget-release.yml b/build/nuget-release.yml index d81119d3..6a14f33a 100644 --- a/build/nuget-release.yml +++ b/build/nuget-release.yml @@ -6,6 +6,10 @@ pr: none parameters: - name: 'Package.Version' type: 'string' + - name: azureServiceConnection + displayName: 'Azure service connection' + type: string + default: 'Azure Codit-Arcus Service Principal' resources: repositories: @@ -15,7 +19,6 @@ resources: endpoint: arcus-azure variables: - - group: 'Arcus Observability - Integration Testing' - group: 'Build Configuration' - template: ./variables/build.yml - template: ./variables/test.yml @@ -57,78 +60,27 @@ stages: dependsOn: Build condition: succeeded() jobs: - - job: UnitTests - displayName: 'Run unit tests' - pool: - vmImage: '$(Vm.Image)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download build artifacts' - inputs: - artifact: 'Build' - path: '$(Build.SourcesDirectory)' - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.PreviousVersion)' - - template: test/run-unit-tests.yml@templates - parameters: - dotnetSdkVersion: '$(DotNet.Sdk.Version)' - includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) - projectName: '$(Project).Tests.Unit' + - template: templates/run-unit-tests.yml - stage: IntegrationTests displayName: Integration Tests dependsOn: Build condition: succeeded() jobs: - - job: IntegrationTests - displayName: 'Run integration tests' - pool: - vmImage: '$(Vm.Image)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download build artifacts' - inputs: - artifact: 'Build' - path: '$(Build.SourcesDirectory)' - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.PreviousVersion)' - - template: test/run-integration-tests.yml@templates - parameters: - dotnetSdkVersion: '$(DotNet.Sdk.Version)' - includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) - projectName: '$(Project).Tests.Integration' - category: 'Integration' + - template: templates/run-self-contained-integration-tests.yml + parameters: + azureServiceConnection: '${{ parameters.azureServiceConnection }}' - stage: DockerTests displayName: Docker Tests dependsOn: Build condition: succeeded() jobs: - - job: DockerTests - displayName: 'Run Docker tests' - pool: - vmImage: '$(Vm.Image)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download build artifacts' - inputs: - artifact: 'Build' - path: '$(Build.SourcesDirectory)' - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.PreviousVersion)' - - template: templates/run-docker-integration-tests.yml - parameters: - dockerProjectName: '$(Project).Tests.Runtimes.AzureFunction' - httpPort: '$(AzureFunctions.HttpPort)' + - template: templates/run-docker-integration-tests.yml + parameters: + dockerProjectName: '$(Project).Tests.Runtimes.AzureFunction' + httpPort: '$(AzureFunctions.HttpPort)' + azureServiceConnection: '${{ parameters.azureServiceConnection }}' - stage: Release displayName: 'Release to NuGet.org' diff --git a/build/templates/deploy-test-resources.bicep b/build/templates/deploy-test-resources.bicep new file mode 100644 index 00000000..de12ca69 --- /dev/null +++ b/build/templates/deploy-test-resources.bicep @@ -0,0 +1,95 @@ +// Define the location for the deployment of the components. +param location string + +// Define the name of the resource group where the components will be deployed. +param resourceGroupName string + +// Define the name of the Application Insights component. +param appInsightsName string + +// Define the name of the secret that will store the Application Insights Instrumentation Key. +param instrumentationKey_secretName string + +// Define the name of the secret that will store the Application Insights Application ID. +param applicationId_secretName string + +// Define the name of the Key Vault. +param keyVaultName string + +// Define the Service Principal ID that needs access full access to the deployed resource group. +param servicePrincipal_objectId string + +targetScope='subscription' + +module resourceGroup 'br/public:avm/res/resources/resource-group:0.2.3' = { + name: 'resourceGroupDeployment' + params: { + name: resourceGroupName + location: location + } +} + +resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + name: resourceGroupName +} + +module workspace 'br/public:avm/res/operational-insights/workspace:0.3.4' = { + name: 'workspaceDeployment' + dependsOn: [ + resourceGroup + ] + scope: rg + params: { + name: 'arcus-observability-dev-we-workspace' + location: location + } +} + +module component 'br/public:avm/res/insights/component:0.3.0' = { + name: 'componentDeployment' + dependsOn: [ + resourceGroup + ] + scope: rg + params: { + name: appInsightsName + workspaceResourceId: workspace.outputs.resourceId + location: location + roleAssignments: [ + { + principalId: servicePrincipal_objectId + roleDefinitionIdOrName: 'Log Analytics Contributor' + } + ] + } +} + +module vault 'br/public:avm/res/key-vault/vault:0.6.1' = { + name: 'vaultDeployment' + dependsOn: [ + resourceGroup + ] + scope: rg + params: { + name: keyVaultName + location: location + roleAssignments: [ + { + principalId: servicePrincipal_objectId + roleDefinitionIdOrName: 'Key Vault Secrets officer' + } + ] + secrets: [ + { + name: instrumentationKey_secretName + value: component.outputs.instrumentationKey + } + { + name: applicationId_secretName + value: component.outputs.applicationId + } + ] + } +} + +output ApplicationInsights_ResourceId string = component.outputs.resourceId diff --git a/build/templates/import-keyvault-secrets.yml b/build/templates/import-keyvault-secrets.yml new file mode 100644 index 00000000..d61773aa --- /dev/null +++ b/build/templates/import-keyvault-secrets.yml @@ -0,0 +1,31 @@ +parameters: + azureServiceConnection: '' + +steps: + - task: AzureCLI@2 + displayName: 'Import secrets from Azure Key Vault' + env: + KeyVault_Name: $(KeyVault.Name) + ApplicationInsights_InstrumentationKey_SecretName: $(ApplicationInsights.InstrumentationKey.SecretName) + ApplicationInsights_ApiKey_SecretName: $(ApplicationInsights.ApiKey.SecretName) + ApplicationInsights_ApplicationId_SecretName: $(ApplicationInsights.ApplicationId.SecretName) + inputs: + azureSubscription: '${{ parameters.azureServiceConnection }}' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module -Name Arcus.Scripting.DevOps -AllowClobber + + $secretNames = @( + $env:ApplicationInsights_InstrumentationKey_SecretName, + $env:ApplicationInsights_ApiKey_SecretName, + $env:ApplicationInsights_ApplicationId_SecretName) + + $secretNames | ForEach-Object { + $secretName = $_ + $variableName = $secretName -replace '-','.' + Write-Host "Importing Azure Key vault secret '$secretName' as Azure DevOps pipeline variable '$variableName'" + + $secret = az keyvault secret show --name $secretName --vault-name $env:KeyVault_Name | ConvertFrom-Json + Set-AzDevOpsVariable $variableName -Value $secret.value -AsSecret } \ No newline at end of file diff --git a/build/templates/run-docker-integration-tests.yml b/build/templates/run-docker-integration-tests.yml index 97e817ad..007d2370 100644 --- a/build/templates/run-docker-integration-tests.yml +++ b/build/templates/run-docker-integration-tests.yml @@ -1,54 +1,82 @@ parameters: dockerProjectName: '' httpPort: '' + azureServiceConnection: '' -steps: - - bash: | - if [ -z "$PROJECT_NAME" ]; then - echo "##vso[task.logissue type=error;]Missing template parameter \"dockerProjectName\"" - echo "##vso[task.complete result=Failed;]" - fi - if [ -z "$HTTP_PORT" ]; then - echo "##vso[task.logissue type=error;]Missing template parameter \"httpPort\"" - echo "##vso[task.complete result=Failed;]" - fi - env: - PROJECT_NAME: ${{ parameters.dockerProjectName }} - HTTP_PORT: ${{ parameters.httpPort }} - - task: UseDotNet@2 - displayName: 'Import .NET Core SDK ($(DotNet.Sdk.Version))' - inputs: - packageType: 'sdk' - version: '$(DotNet.Sdk.Version)' - includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) - - task: Docker@1 - displayName: 'Build Docker image from ${{ parameters.dockerProjectName }}' - inputs: - dockerFile: src/${{ parameters.dockerProjectName }}/Dockerfile - imageName: '${{ parameters.dockerProjectName }}:$(Build.BuildId)' - useDefaultContext: false - buildContext: src - - task: Docker@1 - displayName: 'Run new project Docker image from ${{ parameters.dockerProjectName }}' - inputs: - command: 'Run an image' - imageName: '${{ parameters.dockerProjectName }}:$(Build.BuildId)' - containerName: '${{ parameters.dockerProjectName }}' - ports: '${{ parameters.httpPort }}:80' - envVars: | - APPINSIGHTS_INSTRUMENTATIONKEY=$(ApplicationInsights.InstrumentationKey) - AzureWebJobsStorage=$(Arcus.AzureFunctions.AzureWebJobsStorage) - - template: test/run-integration-tests.yml@templates - parameters: - dotnetSdkVersion: '$(DotNet.Sdk.Version)' - includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) - projectName: '$(Project).Tests.Integration' - category: 'Docker' - - task: Bash@3 - inputs: - targetType: 'inline' - script: | - docker logs ${{ parameters.dockerProjectName }} - failOnStderr: true - displayName: Show ${{ parameters.dockerProjectName }} logs - condition: always() +jobs: + - job: DockerTests + displayName: 'Run Docker tests' + pool: + vmImage: '$(Vm.Image)' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download build artifacts' + inputs: + artifact: 'Build' + path: '$(Build.SourcesDirectory)' + + - bash: | + if [ -z "$PROJECT_NAME" ]; then + echo "##vso[task.logissue type=error;]Missing template parameter \"dockerProjectName\"" + echo "##vso[task.complete result=Failed;]" + fi + if [ -z "$HTTP_PORT" ]; then + echo "##vso[task.logissue type=error;]Missing template parameter \"httpPort\"" + echo "##vso[task.complete result=Failed;]" + fi + env: + PROJECT_NAME: ${{ parameters.dockerProjectName }} + HTTP_PORT: ${{ parameters.httpPort }} + + - task: UseDotNet@2 + displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' + inputs: + packageType: 'sdk' + version: '$(DotNet.Sdk.PreviousVersion)' + + - task: UseDotNet@2 + displayName: 'Import .NET Core SDK ($(DotNet.Sdk.Version))' + inputs: + packageType: 'sdk' + version: '$(DotNet.Sdk.Version)' + includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) + + - template: import-keyvault-secrets.yml + parameters: + azureServiceConnection: '${{ parameters.azureServiceConnection }}' + + - task: Docker@1 + displayName: 'Build Docker image from ${{ parameters.dockerProjectName }}' + inputs: + dockerFile: src/${{ parameters.dockerProjectName }}/Dockerfile + imageName: '${{ parameters.dockerProjectName }}:$(Build.BuildId)' + useDefaultContext: false + buildContext: src + + - task: Docker@1 + displayName: 'Run new project Docker image from ${{ parameters.dockerProjectName }}' + inputs: + command: 'Run an image' + imageName: '${{ parameters.dockerProjectName }}:$(Build.BuildId)' + containerName: '${{ parameters.dockerProjectName }}' + ports: '${{ parameters.httpPort }}:80' + envVars: | + APPINSIGHTS_INSTRUMENTATIONKEY=$(ApplicationInsights_InstrumentationKey) + AzureWebJobsStorage=$(Arcus.AzureFunctions.AzureWebJobsStorage) + + - template: test/run-integration-tests.yml@templates + parameters: + dotnetSdkVersion: '$(DotNet.Sdk.Version)' + includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) + projectName: '$(Project).Tests.Integration' + category: 'Docker' + + - task: Bash@3 + inputs: + targetType: 'inline' + script: | + docker logs ${{ parameters.dockerProjectName }} + failOnStderr: true + displayName: Show ${{ parameters.dockerProjectName }} logs + condition: always() + \ No newline at end of file diff --git a/build/templates/run-self-contained-integration-tests.yml b/build/templates/run-self-contained-integration-tests.yml new file mode 100644 index 00000000..6cf3e5d3 --- /dev/null +++ b/build/templates/run-self-contained-integration-tests.yml @@ -0,0 +1,31 @@ +parameters: + azureServiceConnection: '' + +jobs: + - job: IntegrationTests + displayName: 'Run integration tests' + pool: + vmImage: '$(Vm.Image)' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download build artifacts' + inputs: + artifact: 'Build' + path: '$(Build.SourcesDirectory)' + + - task: UseDotNet@2 + displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' + inputs: + packageType: 'sdk' + version: '$(DotNet.Sdk.PreviousVersion)' + + - template: import-keyvault-secrets.yml + parameters: + azureServiceConnection: '${{ parameters.azureServiceConnection }}' + + - template: test/run-integration-tests.yml@templates + parameters: + dotnetSdkVersion: '$(DotNet.Sdk.Version)' + includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) + projectName: '$(Project).Tests.Integration' + category: 'Integration' \ No newline at end of file diff --git a/build/templates/run-unit-tests.yml b/build/templates/run-unit-tests.yml new file mode 100644 index 00000000..8f3a4dbd --- /dev/null +++ b/build/templates/run-unit-tests.yml @@ -0,0 +1,23 @@ +jobs: + - job: UnitTests + displayName: 'Run unit tests' + pool: + vmImage: '$(Vm.Image)' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download build artifacts' + inputs: + artifact: 'Build' + path: '$(Build.SourcesDirectory)' + + - task: UseDotNet@2 + displayName: 'Import .NET Core SDK ($(DotNet.Sdk.PreviousVersion))' + inputs: + packageType: 'sdk' + version: '$(DotNet.Sdk.PreviousVersion)' + + - template: test/run-unit-tests.yml@templates + parameters: + dotnetSdkVersion: '$(DotNet.Sdk.Version)' + includePreviewVersions: $(DotNet.Sdk.IncludePreviewVersions) + projectName: '$(Project).Tests.Unit' \ No newline at end of file diff --git a/build/variables/test.yml b/build/variables/test.yml index f6ec8be2..65546bd8 100644 --- a/build/variables/test.yml +++ b/build/variables/test.yml @@ -1,2 +1,7 @@ variables: - AzureFunctions.HttpPort: 5000 \ No newline at end of file + AzureFunctions.HttpPort: 5000 + KeyVault.Name: 'arcus-observability-kv' + ApplicationInsights.Name: 'arcus-observability-dev-we-app-insights' + ApplicationInsights.ApplicationId.SecretName: ApplicationInsights-ApplicationId + ApplicationInsights.InstrumentationKey.SecretName: ApplicationInsights-InstrumentationKey + ApplicationInsights.ApiKey.SecretName: ApplicationInsights-ApiKey \ No newline at end of file