main-build-and-deploy.yml

name: Main branch Build and Deployment
on:
  pull_request:
    types: [closed]
    branches:
      - main
jobs:
  cancel-incomplete-validations:
    name: Cancel all incomplete PR-validations for closed PR (further validation on closed PRs are irrelevant)
    permissions:
      actions: write
    runs-on: ubuntu-22.04
    concurrency:
      group: PR validation-${{ github.event.pull_request.number }}
      cancel-in-progress: true
    steps:
    # Execution of this job (step) will cancel all incomplete PR-validation runs on the closed PR through GHA concurrency.
    - name: Cancelling all PR validation runs for PR ${{ github.event.pull_request.number }}
      run: |
        echo "Done."
  on-merge:
    name: Skip on unmerged
    if: github.event.pull_request.merged == true
    runs-on: ubuntu-22.04
    steps:
    - name: Confirm execution
      shell: bash
      run: |
        echo "PR merge detected and deployment wanted."
  on-deploy:
    name: Skip on 'no-deploy' PRs
    needs: on-merge
    if: '!contains(github.event.pull_request.labels.*.name, ''no-deploy'')'
    runs-on: ubuntu-22.04
    steps:
    - name: Confirm execution
      shell: bash
      run: |
        echo "PR merge detected and deployment wanted."
  commit-deps-lock-updates:
    name: Commit dependency lock updates as proposed during PR validation
    concurrency:
      group: ${{ github.workflow }}-commit-deps-lock-updates
      cancel-in-progress: true
    runs-on: ubuntu-22.04
    needs:
      - on-merge
    permissions:
      contents: write
    outputs:
      latest-commit-sha: ${{ steps.store-output.outputs.latest-commit-sha }}
      git-describe: ${{ steps.store-output.outputs.git-describe }}
    env:
      latest_commit_sha:
    steps:
      - name: create token for committing and pushing as agr-github-actions app
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ secrets.GH_ACTIONS_APP_ID }}
          private-key: ${{ secrets.GH_ACTIONS_APP_PRIVATE_KEY }}
      - name: Get GitHub App User ID
        id: app-user-id
        run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT"
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ steps.app-token.outputs.token }}
          persist-credentials: true
      - name: Store checkout sha in env variable
        run: |
          echo "latest_commit_sha=$(git log -1 --format=%H)" >> "$GITHUB_ENV"
      - name: Download updated dependencies lock files bundle
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        uses: dawidd6/action-download-artifact@v8
        with:
          name: deps_lock_files_bundle
          pr: ${{ github.event.pull_request.number }}
          workflow: PR-validation.yml
          workflow_conclusion: success
          workflow_search: false
          allow_forks: false
          if_no_artifact_found: fail
      - name: Unpack the bundle
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        run: |
          tar -xzv -f deps-lock-files.tar.gz
      - name: commit dependency lock file changes
        id: deps-lock-commit
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          branch: ${{ github.base_ref }}
          commit_user_name: ${{ steps.app-token.outputs.app-slug }}[bot]
          commit_user_email: ${{ steps.app-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com
          commit_author: ${{ steps.app-token.outputs.app-slug }}[bot] <${{ steps.app-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com>
          commit_message: Auto-updated deps lock files [skip actions]
          file_pattern: '*requirements.txt *package-lock.json'
          disable_globbing: true
          skip_checkout: true
          skip_fetch: true
      - name: Store commit sha in env variable (if updated)
        if: steps.deps-lock-commit.outputs.commit_hash
        run: |
          echo "latest_commit_sha=${{ steps.deps-lock-commit.outputs.commit_hash }}" >> "$GITHUB_ENV"
      - name: store latest_commit_sha job output
        id: store-output
        run: |
          echo "latest-commit-sha=${{ env.latest_commit_sha }}" >> "$GITHUB_OUTPUT"
          echo "git-describe=$(git describe --tags)" >> $GITHUB_OUTPUT
  build-pavi-shared-aws-module:
    name: Build pavi_shared_aws package
    runs-on: ubuntu-22.04
    concurrency:
      group: ${{ github.workflow }}-build-pavi-shared-aws-module
      cancel-in-progress: true
    needs: [commit-deps-lock-updates, on-deploy]
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            shared_aws/py_package/
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version-file: "shared_aws/py_package/pyproject.toml"
      - name: Build the pavi_shared_aws package
        working-directory: ./shared_aws/py_package
        run: |
          make clean build
      - name: Upload package as artifact
        uses: actions/upload-artifact@v4
        with:
          name: shared_aws_py_package
          path: shared_aws/py_package/dist/pavi_shared_aws-0.0.0-py3-none-any.whl
  deploy-shared-aws-infra:
    name: Deploy/update shared PAVI AWS infrastructure
    concurrency:
      group: ${{ github.workflow }}-deploy-shared-aws-infra
      cancel-in-progress: false
    needs:
      - commit-deps-lock-updates
      - on-deploy
      - build-pavi-shared-aws-module
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: shared_aws/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: 'shared_aws/aws_infra/.nvmrc'
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "shared_aws/aws_infra/pyproject.toml"
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-deploy
        aws-region: us-east-1
    - name: CDK validations (resource assertions and cdk diff)
      run: make validate-stack
    - name: Deploy CDK Stack
      run: make deploy-stack ADD_CDK_ARGS="--require-approval never"
  pipeline-deploy-aws-infra:
    name: Deploy/update AWS infrastructure for pipeline
    concurrency:
      group: ${{ github.workflow }}-pipeline-deploy-aws-infra
      cancel-in-progress: false
    needs:
      - commit-deps-lock-updates
      - on-deploy
      - build-pavi-shared-aws-module
      - deploy-shared-aws-infra
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: pipeline/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "pipeline/aws_infra/.nvmrc"
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "pipeline/aws_infra/pyproject.toml"
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-deploy
        aws-region: us-east-1
    - name: CDK validations (resource assertions and cdk diff)
      run: make validate-stack
    - name: Deploy CDK Stack
      run: make deploy ADD_CDK_ARGS="--require-approval never"
  api-deploy-image-repo:
    name: Deploy/update container image repository stack for API
    concurrency:
      group: ${{ github.workflow }}-api-deploy-image-repo
      cancel-in-progress: false
    needs:
      - commit-deps-lock-updates
      - on-deploy
      - build-pavi-shared-aws-module
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: api/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "api/aws_infra/.nvmrc"
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "api/aws_infra/pyproject.toml"
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-deploy
        aws-region: us-east-1
    - name: CDK validations (resource assertions and cdk diff)
      run: make validate-image-stack
    - name: Deploy CDK stack
      run: make deploy-image-stack ADD_CDK_ARGS="--require-approval never"
  webui-deploy-image-repo:
    name: Deploy/update container image repository stack for web UI
    concurrency:
      group: ${{ github.workflow }}-webui-deploy-image-repo
      cancel-in-progress: false
    needs:
      - commit-deps-lock-updates
      - on-deploy
      - build-pavi-shared-aws-module
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: webui/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "webui/aws_infra/.nvmrc"
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "webui/aws_infra/pyproject.toml"
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-deploy
        aws-region: us-east-1
    - name: CDK validations (resource assertions and cdk diff)
      run: make validate-image-stack
    - name: Deploy CDK stack
      run: make deploy-image-stack ADD_CDK_ARGS="--require-approval never"
  pipeline-seq-retrieval-build-and-push-docker-image:
    concurrency:
      group: ${{ github.workflow }}-pipeline-seq-retrieval-build-and-push-docker-image
      cancel-in-progress: false
    needs: [on-deploy, pipeline-deploy-aws-infra, commit-deps-lock-updates]
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    env:
      tagname: ${{ needs.commit-deps-lock-updates.outputs.git-describe }}
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
        sparse-checkout: |
          pipeline/seq_retrieval/
    # This step will configure environment variables to be used by all steps
    # involving AWS interaction further down
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-build-image-pipeline-seq-retrieval
        aws-region: us-east-1
    - name: Amazon ECR login
      id: login-ecr
      uses: aws-actions/amazon-ecr-login@v2
    - name: Build and push container image
      uses: docker/build-push-action@v6
      env:
          DOCKER_BUILD_SUMMARY: false
      with:
        context: ./pipeline/seq_retrieval/
        push: true
        tags: |
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{ env.tagname }}
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{ github.event.pull_request.base.ref }}
        platforms: linux/amd64
  pipeline-alignment-build-and-push-docker-image:
    concurrency:
      group: ${{ github.workflow }}-pipeline-alignment-build-and-push-docker-image
      cancel-in-progress: false
    needs: [on-deploy, pipeline-deploy-aws-infra, commit-deps-lock-updates]
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    env:
      tagname: ${{ needs.commit-deps-lock-updates.outputs.git-describe }}
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
        sparse-checkout: |
          pipeline/alignment/
    # This step will configure environment variables to be used by all steps
    # involving AWS interaction further down
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-build-image-pipeline-seq-retrieval
        aws-region: us-east-1
    - name: Amazon ECR login
      id: login-ecr
      uses: aws-actions/amazon-ecr-login@v2
    - name: Build and push container image
      uses: docker/build-push-action@v6
      env:
          DOCKER_BUILD_SUMMARY: false
      with:
        context: ./pipeline/alignment/
        push: true
        tags: |
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{ env.tagname }}
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{ github.event.pull_request.base.ref }}
        platforms: linux/amd64
  api-build-and-push-docker-image:
    concurrency:
      group: ${{ github.workflow }}-api-build-and-push-docker-image
      cancel-in-progress: false
    needs: [on-deploy, api-deploy-image-repo, commit-deps-lock-updates]
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    env:
      tagname: ${{ needs.commit-deps-lock-updates.outputs.git-describe }}
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    # This step will configure environment variables to be used by all steps
    # involving AWS interaction further down
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-build-image-api
        aws-region: us-east-1
    - name: Amazon ECR login
      id: login-ecr
      uses: aws-actions/amazon-ecr-login@v2
    - name: Store AWS credentials in (AWS CLI compatible) config and credentials files
      run: |
        echo "[default]" > aws.config
        echo "region = $AWS_REGION" >> aws.config
        echo "[default]" > aws.credentials
        echo "aws_access_key_id = $AWS_ACCESS_KEY_ID" >> aws.credentials
        echo "aws_secret_access_key = $AWS_SECRET_ACCESS_KEY" >> aws.credentials
        echo "aws_session_token = $AWS_SESSION_TOKEN" >> aws.credentials
    - name: Build and push container image
      uses: docker/build-push-action@v6
      env:
          DOCKER_BUILD_SUMMARY: false
      with:
        context: ./
        file: api/Dockerfile
        push: true
        secrets: |
          AWS_PROFILE=default
        secret-files: |
          AWS_SHARED_CREDENTIALS_FILE=aws.credentials
          AWS_CONFIG_FILE=aws.config
        tags: |
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/api:${{ env.tagname }}
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/api:${{ github.event.pull_request.base.ref }}
        platforms: linux/amd64
  webui-build-and-push-docker-image:
    concurrency:
      group: ${{ github.workflow }}-webui-build-and-push-docker-image
      cancel-in-progress: false
    needs: [on-deploy, webui-deploy-image-repo, commit-deps-lock-updates]
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    env:
      tagname: ${{ needs.commit-deps-lock-updates.outputs.git-describe }}
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Report node version stored in .nvmrc
      id: nvmrc-node-version
      run: |
        {
          echo 'content<<EOF'
          cat webui/.nvmrc
          echo EOF
        } >> "$GITHUB_OUTPUT"
    # This step will configure environment variables to be used by all steps
    # involving AWS interaction further down
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-build-image-webui
        aws-region: us-east-1
    - name: Amazon ECR login
      id: login-ecr
      uses: aws-actions/amazon-ecr-login@v2
    - name: Build and push container image
      uses: docker/build-push-action@v6
      env:
          DOCKER_BUILD_SUMMARY: false
      with:
        build-args: NODE_VERSION=${{steps.nvmrc-node-version.outputs.content}}
        context: ./webui/
        push: true
        tags: |
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/webui:${{ env.tagname }}
          ${{ steps.login-ecr.outputs.registry }}/agr_pavi/webui:${{ github.event.pull_request.base.ref }}
        platforms: linux/amd64
  deploy-application:
    name: Deploy application (version) (API and WebUI)
    concurrency:
      group: ${{ github.workflow }}-deploy-application
      cancel-in-progress: false
    needs:
      - on-deploy
      - commit-deps-lock-updates
      - build-pavi-shared-aws-module
      - api-build-and-push-docker-image
      - pipeline-alignment-build-and-push-docker-image
      - pipeline-seq-retrieval-build-and-push-docker-image
      - webui-build-and-push-docker-image
      - deploy-shared-aws-infra
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    env:
      tagname: ${{ needs.commit-deps-lock-updates.outputs.git-describe }}
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }}
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "shared_aws/aws_infra/.nvmrc"
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-app-cdk-deploy
        aws-region: us-east-1
    # API validation and deployment
    - name: Setup Python for API deployment
      uses: actions/setup-python@v5
      with:
        python-version-file: "api/aws_infra/pyproject.toml"
    - name: API CDK validations (resource assertions and cdk diff)
      working-directory: api/aws_infra
      run: make validate-application-stack validate-environment-stack PAVI_DEPLOY_VERSION_LABEL="${{ env.tagname }}" PAVI_IMAGE_TAG="${{ env.tagname }}"
    - name: Deploy API application (and version)
      working-directory: api/aws_infra
      run: make deploy-application PAVI_DEPLOY_VERSION_LABEL="${{ env.tagname }}" ADD_CDK_ARGS="--require-approval never"
    - name: Deploy API to main environment
      working-directory: api/aws_infra
      run: make deploy-environment PAVI_DEPLOY_VERSION_LABEL="${{ env.tagname }}" PAVI_IMAGE_TAG="${{ env.tagname }}" \
            EB_ENV_CDK_STACK_NAME=PaviApiEbMainStack ADD_CDK_ARGS="--require-approval never"
    # webUI validation and deployment
    - name: Setup Python for webUI deployment
      uses: actions/setup-python@v5
      with:
        python-version-file: "webui/aws_infra/pyproject.toml"
    - name: webUI CDK validations (resource assertions and cdk diff)
      working-directory: webui/aws_infra
      run: make validate-application-stack validate-environment-stack PAVI_API_STACK_NAME="PaviApiEbMainStack" \
            PAVI_DEPLOY_VERSION_LABEL="${{ env.tagname }}" PAVI_IMAGE_TAG="${{ env.tagname }}"
    - name: Deploy webUI application (and version)
      working-directory: webui/aws_infra
      run: make deploy-application PAVI_DEPLOY_VERSION_LABEL="${{ env.tagname }}" ADD_CDK_ARGS="--require-approval never"
    - name: Deploy webUI to main environment
      working-directory: webui/aws_infra
      run: make deploy-environment PAVI_API_STACK_NAME="PaviApiEbMainStack" \
            PAVI_DEPLOY_VERSION_LABEL="${{ env.tagname }}" PAVI_IMAGE_TAG="${{ env.tagname }}" \
            EB_ENV_CDK_STACK_NAME=PaviWebUiEbMainStack ADD_CDK_ARGS="--require-approval never"