Azure Batch CI/CD for commit f0c7d464e23f0a79f02ee9302075d304a9af0580 #430
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Azure Batch CI/CD | |
| run-name: "Azure Batch CI/CD for commit ${{ github.sha }}" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "indexer/**" | |
| - ".github/workflows/batch_pipeline.yaml" | |
| pull_request: | |
| branches: | |
| - main | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - "indexer/**" | |
| - ".github/workflows/batch_pipeline.yaml" | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| IMAGE_NAME: "indexer" | |
| IMAGE_TAG: ${{ github.sha }} | |
| jobs: | |
| pre_deploy: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main') | |
| steps: | |
| - name: Invalid branch | |
| if: github.event_name == 'workflow_dispatch' && github.event.ref != 'refs/heads/main' | |
| run: | | |
| echo "Invalid branch" | |
| exit 1 | |
| indexer_ci: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' && github.event.action != 'closed' | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: "Do nothing" | |
| run: echo "Doing nothing for now" | |
| buildImage: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main') | |
| needs: pre_deploy | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: azure/[email protected] | |
| name: Azure login | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| # We check if the image already exists in the ACR, if it does we skip the build | |
| - name: Check if image exists | |
| id: check-if-image-exists | |
| if: github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main' | |
| run: | | |
| set -e | |
| EXIT_CODE=0 | |
| az acr repository show --name ${{ secrets.AZURE_CONTAINER_REGISTRY }} --image ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} > /dev/null || EXIT_CODE=$? | |
| if [ $EXIT_CODE -eq 0 ]; then | |
| echo "Image already exists in ACR, skipping build" | |
| echo "::set-output name=skip_build::true" | |
| fi | |
| exit 0 | |
| - name: Build and push image to ACR | |
| working-directory: ./indexer | |
| if: github.event_name == 'push' || steps.check-if-image-exists.outputs.skip_build != 'true' | |
| # TODO: Secrets are not working in the build command, we need to fix this | |
| run: az acr build --image ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} --image ${{ env.IMAGE_NAME }}:latest --registry ${{ secrets.AZURE_CONTAINER_REGISTRY }} -g ${{ secrets.ACR_RESOURCE_GROUP }} --secret-build-arg AZURE_SEARCH_API_KEY=${{ secrets.AZURE_SEARCH_SERVICE_API_KEY }} --secret-build-arg AZURE_SEARCH_SERVICE_NAME=${{ secrets.AZURE_SEARCH_SERVICE_NAME }} --secret-build-arg AZURE_SEARCH_INDEX_NAME=indexer --secret-build-arg AZURE_SEARCH_SEMANTIC_CONFIG_NAME=${{ secrets.AZURE_SEARCH_SEMANTIC_CONFIG_NAME }} --secret-build-arg OPENAI_KEY=${{ secrets.OPENAI_KEY }} --secret-build-arg OPENAI_DEPLOYMENT_NAME=${{ secrets.OPENAI_DEPLOYMENT_NAME}} --secret-build-arg OPENAI_PROJECT_NAME=${{ secrets.OPENAI_PROJECT_NAME}} --secret-build-arg OPENAI_PROJECT_ID=${{ secrets.OPENAI_PROJECT_ID}} --secret-build-arg OPENAI_ORG_NAME=${{ secrets.OPENAI_ORG_NAME}} --secret-build-arg OPENAI_ORG_ID=${{ secrets.OPENAI_ORG_ID}} . | |
| deploy: | |
| runs-on: ubuntu-latest | |
| needs: buildImage | |
| env: | |
| AZURE_BATCH_ENDPOINT: ${{ secrets.BATCH_ACCOUNT_ENDPOINT }} | |
| AZURE_BATCH_ACCESS_KEY: ${{ secrets.BATCH_ACCOUNT_KEY }} | |
| AZURE_BATCH_ACCOUNT: ${{ secrets.BATCH_ACCOUNT_NAME }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: azure/[email protected] | |
| name: Azure login | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Creates Batch Task Configuration | |
| run: | | |
| echo "{ | |
| \"id\": \"${{ github.sha }}\", | |
| \"displayName\": \"Indexer Task for commit ${{ github.sha }}\", | |
| \"commandLine\": \"\", | |
| \"containerSettings\": { | |
| \"imageName\": \"${{ secrets.AZURE_CONTAINER_REGISTRY_SERVER }}/${{ env.IMAGE_NAME }}:latest\", | |
| \"containerRunOptions\": \"--rm --workdir /indexer --env AZURE_SEARCH_API_KEY=${{ secrets.AZURE_SEARCH_SERVICE_API_KEY }} --env AZURE_SEARCH_SERVICE_NAME=${{ secrets.AZURE_SEARCH_SERVICE_NAME }} --env AZURE_SEARCH_INDEX_NAME=${{ secrets.AZURE_SEARCH_INDEX_NAME }} --env AZURE_SEARCH_SEMANTIC_CONFIG_NAME=${{ secrets.AZURE_SEARCH_SEMANTIC_CONFIG_NAME }} --env OPENAI_KEY=${{ secrets.OPENAI_KEY }} --env OPENAI_DEPLOYMENT_NAME=${{ secrets.OPENAI_DEPLOYMENT_NAME}} --env OPENAI_PROJECT_NAME=${{ secrets.OPENAI_PROJECT_NAME}} --env OPENAI_PROJECT_ID=${{ secrets.OPENAI_PROJECT_ID}} --env OPENAI_ORG_NAME=${{ secrets.OPENAI_ORG_NAME}} --env OPENAI_ORG_ID=${{ secrets.OPENAI_ORG_ID }}\", | |
| }, | |
| \"OutputFiles\": [ | |
| { | |
| \"filePattern\": \"../**/*.txt\", | |
| \"destination\": { | |
| \"container\": { | |
| \"containerUrl\": \"${{ secrets.AZURE_STORAGE_CONTAINER_URL }}\", | |
| \"path\": \"logs/${{ github.sha }}\", | |
| \"identityReference\": { | |
| \"resourceId\": \"${{ secrets.BATCH_POOL_IDENTITY_ID }}\", | |
| } | |
| } | |
| }, | |
| \"uploadOptions\": { | |
| \"uploadCondition\": \"taskCompletion\" | |
| } | |
| }, | |
| { | |
| \"filePattern\": \"../**/data/*.json\", | |
| \"destination\": { | |
| \"container\": { | |
| \"containerUrl\": \"${{ secrets.AZURE_STORAGE_CONTAINER_URL }}\", | |
| \"path\": \"logs/${{ github.sha }}/data\", | |
| \"identityReference\": { | |
| \"resourceId\": \"${{ secrets.BATCH_POOL_IDENTITY_ID }}\", | |
| } | |
| } | |
| }, | |
| \"uploadOptions\": { | |
| \"uploadCondition\": \"taskCompletion\" | |
| } | |
| }, | |
| ], | |
| \"UserIdentity\": { | |
| \"autoUser\": { | |
| \"scope\": \"Task\", | |
| \"elevationLevel\": \"admin\" | |
| } | |
| }, | |
| }" > task.json | |
| - name: Check if job exists | |
| id: check-if-job-exists | |
| if: github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main' | |
| run: | | |
| set -e | |
| EXIT_CODE=0 | |
| az batch task show --job-id ${{ secrets.BATCH_JOB_ID }} --task-id ${{ github.sha }} > /dev/null || EXIT_CODE=$? | |
| if [ $EXIT_CODE -eq 0 ]; then | |
| echo "::set-output name=task_exists::true" | |
| fi | |
| exit 0 | |
| - name: Delete Batch Job Task | |
| if: steps.check-if-job-exists.outputs.task_exists == 'true' | |
| run: | | |
| az batch task delete --job-id ${{ secrets.BATCH_JOB_ID }} --task-id ${{ github.sha }} --yes | |
| - name: Deploy to Azure Batch | |
| run: az batch task create --job-id ${{ secrets.BATCH_JOB_ID }} --json-file task.json |