-
Notifications
You must be signed in to change notification settings - Fork 0
162 lines (162 loc) · 8.77 KB
/
batch_pipeline.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
name: Azure Batch CI/CD
run-name: "Azure Batch CI/CD for commit ${{ github.sha }}"
on:
workflow_dispatch:
push:
branches:
- main
paths:
- "indexer/**"
- ".github/workflows/batch_pipeline.yaml"
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
paths:
- "indexer/**"
- ".github/workflows/batch_pipeline.yaml"
permissions:
contents: read
id-token: write
env:
IMAGE_NAME: "indexer"
IMAGE_TAG: ${{ github.sha }}
jobs:
pre_deploy:
runs-on: ubuntu-latest
if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main')
steps:
- name: Invalid branch
if: github.event_name == 'workflow_dispatch' && github.event.ref != 'refs/heads/main'
run: |
echo "Invalid branch"
exit 1
indexer_ci:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request' && github.event.action != 'closed'
steps:
- uses: actions/[email protected]
- name: "Do nothing"
run: echo "Doing nothing for now"
buildImage:
runs-on: ubuntu-latest
if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main')
needs: pre_deploy
steps:
- uses: actions/[email protected]
- uses: azure/[email protected]
name: Azure login
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
# We check if the image already exists in the ACR, if it does we skip the build
- name: Check if image exists
id: check-if-image-exists
if: github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main'
run: |
set -e
EXIT_CODE=0
az acr repository show --name ${{ secrets.AZURE_CONTAINER_REGISTRY }} --image ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} > /dev/null || EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo "Image already exists in ACR, skipping build"
echo "::set-output name=skip_build::true"
fi
exit 0
- name: Build and push image to ACR
working-directory: ./indexer
if: github.event_name == 'push' || steps.check-if-image-exists.outputs.skip_build != 'true'
# TODO: Secrets are not working in the build command, we need to fix this
run:
az acr build . --image ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} \
--image ${{ env.IMAGE_NAME }}:latest \
--registry ${{ secrets.AZURE_CONTAINER_REGISTRY }} \
-g ${{ secrets.ACR_RESOURCE_GROUP }} \
--secret-build-arg AZURE_SEARCH_API_KEY=${{ secrets.AZURE_SEARCH_SERVICE_API_KEY }} \
--secret-build-arg AZURE_SEARCH_SERVICE_NAME=${{ secrets.AZURE_SEARCH_SERVICE_NAME }} \
--secret-build-arg AZURE_SEARCH_INDEX_NAME=indexer \
--secret-build-arg AZURE_SEARCH_SEMANTIC_CONFIG_NAME=${{ secrets.AZURE_SEARCH_SEMANTIC_CONFIG_NAME }} \
--secret-build-arg OPENAI_KEY=${{ secrets.OPENAI_KEY }} \
--secret-build-arg OPENAI_DEPLOYMENT_NAME=${{ secrets.OPENAI_DEPLOYMENT_NAME}} \
--secret-build-arg OPENAI_PROJECT_NAME=${{ secrets.OPENAI_PROJECT_NAME}} \
--secret-build-arg OPENAI_PROJECT_ID=${{ secrets.OPENAI_PROJECT_ID}} \
--secret-build-arg OPENAI_ORG_NAME=${{ secrets.OPENAI_ORG_NAME}} \
--secret-build-arg OPENAI_ORG_ID=${{ secrets.OPENAI_ORG_ID}} \
--secret-build-arg ENABLE_SEMANTIC_SEARCH=${{ secrets.ENABLE_SEMANTIC_SEARCH}}
deploy:
runs-on: ubuntu-latest
needs: buildImage
env:
AZURE_BATCH_ENDPOINT: ${{ secrets.BATCH_ACCOUNT_ENDPOINT }}
AZURE_BATCH_ACCESS_KEY: ${{ secrets.BATCH_ACCOUNT_KEY }}
AZURE_BATCH_ACCOUNT: ${{ secrets.BATCH_ACCOUNT_NAME }}
steps:
- uses: actions/[email protected]
- uses: azure/[email protected]
name: Azure login
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: Creates Batch Task Configuration
run: |
echo "{
\"id\": \"${{ github.sha }}\",
\"displayName\": \"Indexer Task for commit ${{ github.sha }}\",
\"commandLine\": \"\",
\"containerSettings\": {
\"imageName\": \"${{ secrets.AZURE_CONTAINER_REGISTRY_SERVER }}/${{ env.IMAGE_NAME }}:latest\",
\"containerRunOptions\": \"--rm --workdir /indexer --env AZURE_SEARCH_API_KEY=${{ secrets.AZURE_SEARCH_SERVICE_API_KEY }} --env AZURE_SEARCH_SERVICE_NAME=${{ secrets.AZURE_SEARCH_SERVICE_NAME }} --env AZURE_SEARCH_INDEX_NAME=${{ secrets.AZURE_SEARCH_INDEX_NAME }} --env AZURE_SEARCH_SEMANTIC_CONFIG_NAME=${{ secrets.AZURE_SEARCH_SEMANTIC_CONFIG_NAME }} --env OPENAI_KEY=${{ secrets.OPENAI_KEY }} --env OPENAI_DEPLOYMENT_NAME=${{ secrets.OPENAI_DEPLOYMENT_NAME}} --env OPENAI_PROJECT_NAME=${{ secrets.OPENAI_PROJECT_NAME}} --env OPENAI_PROJECT_ID=${{ secrets.OPENAI_PROJECT_ID}} --env OPENAI_ORG_NAME=${{ secrets.OPENAI_ORG_NAME}} --env OPENAI_ORG_ID=${{ secrets.OPENAI_ORG_ID }} --env ENABLE_SEMANTIC_SEARCH=${{ secrets.ENABLE_SEMANTIC_SEARCH}}\",
},
\"OutputFiles\": [
{
\"filePattern\": \"../**/*.txt\",
\"destination\": {
\"container\": {
\"containerUrl\": \"${{ secrets.AZURE_STORAGE_CONTAINER_URL }}\",
\"path\": \"logs/${{ github.sha }}\",
\"identityReference\": {
\"resourceId\": \"${{ secrets.BATCH_POOL_IDENTITY_ID }}\",
}
}
},
\"uploadOptions\": {
\"uploadCondition\": \"taskCompletion\"
}
},
{
\"filePattern\": \"../**/data/*.json\",
\"destination\": {
\"container\": {
\"containerUrl\": \"${{ secrets.AZURE_STORAGE_CONTAINER_URL }}\",
\"path\": \"logs/${{ github.sha }}/data\",
\"identityReference\": {
\"resourceId\": \"${{ secrets.BATCH_POOL_IDENTITY_ID }}\",
}
}
},
\"uploadOptions\": {
\"uploadCondition\": \"taskCompletion\"
}
},
],
\"UserIdentity\": {
\"autoUser\": {
\"scope\": \"Task\",
\"elevationLevel\": \"admin\"
}
},
}" > task.json
- name: Check if job exists
id: check-if-job-exists
if: github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main'
run: |
set -e
EXIT_CODE=0
az batch task show --job-id ${{ secrets.BATCH_JOB_ID }} --task-id ${{ github.sha }} > /dev/null || EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo "::set-output name=task_exists::true"
fi
exit 0
- name: Delete Batch Job Task
if: steps.check-if-job-exists.outputs.task_exists == 'true'
run: |
az batch task delete --job-id ${{ secrets.BATCH_JOB_ID }} --task-id ${{ github.sha }} --yes
- name: Deploy to Azure Batch
run: az batch task create --job-id ${{ secrets.BATCH_JOB_ID }} --json-file task.json