Azure Batch CI/CD for commit 0f2520250f629c5702ebb088cf37aabcf3331d6a #433
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Azure Batch CI/CD | |
| run-name: "Azure Batch CI/CD for commit ${{ github.sha }}" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "indexer/**" | |
| - ".github/workflows/batch_pipeline.yaml" | |
| pull_request: | |
| branches: | |
| - main | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - "indexer/**" | |
| - ".github/workflows/batch_pipeline.yaml" | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| IMAGE_NAME: "indexer" | |
| IMAGE_TAG: ${{ github.sha }} | |
| jobs: | |
| pre_deploy: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main') | |
| steps: | |
| - name: Invalid branch | |
| if: github.event_name == 'workflow_dispatch' && github.event.ref != 'refs/heads/main' | |
| run: | | |
| echo "Invalid branch" | |
| exit 1 | |
| indexer_ci: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' && github.event.action != 'closed' | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: "Do nothing" | |
| run: echo "Doing nothing for now" | |
| buildImage: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main') | |
| needs: pre_deploy | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: azure/[email protected] | |
| name: Azure login | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| # We check if the image already exists in the ACR, if it does we skip the build | |
| - name: Check if image exists | |
| id: check-if-image-exists | |
| if: github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main' | |
| run: | | |
| set -e | |
| EXIT_CODE=0 | |
| az acr repository show --name ${{ secrets.AZURE_CONTAINER_REGISTRY }} --image ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} > /dev/null || EXIT_CODE=$? | |
| if [ $EXIT_CODE -eq 0 ]; then | |
| echo "Image already exists in ACR, skipping build" | |
| echo "::set-output name=skip_build::true" | |
| fi | |
| exit 0 | |
| - name: Build and push image to ACR | |
| working-directory: ./indexer | |
| if: github.event_name == 'push' || steps.check-if-image-exists.outputs.skip_build != 'true' | |
| # TODO: Secrets are not working in the build command, we need to fix this | |
| run: az acr build --image ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} --image ${{ env.IMAGE_NAME }}:latest --registry ${{ secrets.AZURE_CONTAINER_REGISTRY }} -g ${{ secrets.ACR_RESOURCE_GROUP }} --secret-build-arg AZURE_SEARCH_API_KEY=${{ secrets.AZURE_SEARCH_SERVICE_API_KEY }} --secret-build-arg AZURE_SEARCH_SERVICE_NAME=${{ secrets.AZURE_SEARCH_SERVICE_NAME }} --secret-build-arg AZURE_SEARCH_INDEX_NAME=indexer --secret-build-arg AZURE_SEARCH_SEMANTIC_CONFIG_NAME=${{ secrets.AZURE_SEARCH_SEMANTIC_CONFIG_NAME }} --secret-build-arg OPENAI_KEY=${{ secrets.OPENAI_KEY }} --secret-build-arg OPENAI_DEPLOYMENT_NAME=${{ secrets.OPENAI_DEPLOYMENT_NAME}} --secret-build-arg OPENAI_PROJECT_NAME=${{ secrets.OPENAI_PROJECT_NAME}} --secret-build-arg OPENAI_PROJECT_ID=${{ secrets.OPENAI_PROJECT_ID}} --secret-build-arg OPENAI_ORG_NAME=${{ secrets.OPENAI_ORG_NAME}} --secret-build-arg OPENAI_ORG_ID=${{ secrets.OPENAI_ORG_ID}} . | |
| deploy: | |
| runs-on: ubuntu-latest | |
| needs: buildImage | |
| env: | |
| AZURE_BATCH_ENDPOINT: ${{ secrets.BATCH_ACCOUNT_ENDPOINT }} | |
| AZURE_BATCH_ACCESS_KEY: ${{ secrets.BATCH_ACCOUNT_KEY }} | |
| AZURE_BATCH_ACCOUNT: ${{ secrets.BATCH_ACCOUNT_NAME }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: azure/[email protected] | |
| name: Azure login | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Creates Batch Task Configuration | |
| run: | | |
| echo "{ | |
| \"id\": \"${{ github.sha }}\", | |
| \"displayName\": \"Indexer Task for commit ${{ github.sha }}\", | |
| \"commandLine\": \"\", | |
| \"containerSettings\": { | |
| \"imageName\": \"${{ secrets.AZURE_CONTAINER_REGISTRY_SERVER }}/${{ env.IMAGE_NAME }}:latest\", | |
| \"containerRunOptions\": \"--rm --workdir /indexer --env AZURE_SEARCH_API_KEY=${{ secrets.AZURE_SEARCH_SERVICE_API_KEY }} --env AZURE_SEARCH_SERVICE_NAME=${{ secrets.AZURE_SEARCH_SERVICE_NAME }} --env AZURE_SEARCH_INDEX_NAME=${{ secrets.AZURE_SEARCH_INDEX_NAME }} --env AZURE_SEARCH_SEMANTIC_CONFIG_NAME=${{ secrets.AZURE_SEARCH_SEMANTIC_CONFIG_NAME }} --env OPENAI_KEY=${{ secrets.OPENAI_KEY }} --env OPENAI_DEPLOYMENT_NAME=${{ secrets.OPENAI_DEPLOYMENT_NAME}} --env OPENAI_PROJECT_NAME=${{ secrets.OPENAI_PROJECT_NAME}} --env OPENAI_PROJECT_ID=${{ secrets.OPENAI_PROJECT_ID}} --env OPENAI_ORG_NAME=${{ secrets.OPENAI_ORG_NAME}} --env OPENAI_ORG_ID=${{ secrets.OPENAI_ORG_ID }}\", | |
| }, | |
| \"OutputFiles\": [ | |
| { | |
| \"filePattern\": \"../**/*.txt\", | |
| \"destination\": { | |
| \"container\": { | |
| \"containerUrl\": \"${{ secrets.AZURE_STORAGE_CONTAINER_URL }}\", | |
| \"path\": \"logs/${{ github.sha }}\", | |
| \"identityReference\": { | |
| \"resourceId\": \"${{ secrets.BATCH_POOL_IDENTITY_ID }}\", | |
| } | |
| } | |
| }, | |
| \"uploadOptions\": { | |
| \"uploadCondition\": \"taskCompletion\" | |
| } | |
| }, | |
| { | |
| \"filePattern\": \"../**/data/*.json\", | |
| \"destination\": { | |
| \"container\": { | |
| \"containerUrl\": \"${{ secrets.AZURE_STORAGE_CONTAINER_URL }}\", | |
| \"path\": \"logs/${{ github.sha }}/data\", | |
| \"identityReference\": { | |
| \"resourceId\": \"${{ secrets.BATCH_POOL_IDENTITY_ID }}\", | |
| } | |
| } | |
| }, | |
| \"uploadOptions\": { | |
| \"uploadCondition\": \"taskCompletion\" | |
| } | |
| }, | |
| ], | |
| \"UserIdentity\": { | |
| \"autoUser\": { | |
| \"scope\": \"Task\", | |
| \"elevationLevel\": \"admin\" | |
| } | |
| }, | |
| }" > task.json | |
| - name: Check if job exists | |
| id: check-if-job-exists | |
| if: github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main' | |
| run: | | |
| set -e | |
| EXIT_CODE=0 | |
| az batch task show --job-id ${{ secrets.BATCH_JOB_ID }} --task-id ${{ github.sha }} > /dev/null || EXIT_CODE=$? | |
| if [ $EXIT_CODE -eq 0 ]; then | |
| echo "::set-output name=task_exists::true" | |
| fi | |
| exit 0 | |
| - name: Delete Batch Job Task | |
| if: steps.check-if-job-exists.outputs.task_exists == 'true' | |
| run: | | |
| az batch task delete --job-id ${{ secrets.BATCH_JOB_ID }} --task-id ${{ github.sha }} --yes | |
| - name: Deploy to Azure Batch | |
| run: az batch task create --job-id ${{ secrets.BATCH_JOB_ID }} --json-file task.json |