PR-validation.yml

name: PR validation
on:
  pull_request:
    types: [synchronize, opened, reopened, edited, labeled, unlabeled]
    branches:
      - main
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true
jobs:
  container-build-context:
    name: define container image build context variables
    runs-on: ubuntu-22.04
    outputs:
      tagname: ${{ steps.store_tagname.outputs.tagname }}
    steps:
    - name: Store image tag in env
      id: store_tagname
      shell: bash
      run: |
        echo "tagname=PR-${{ github.event.pull_request.number }}-validation" >> $GITHUB_OUTPUT
  pipeline-seq-retrieval-update-dependency-lock-files:
    name: pipeline/seq_retrieval update and upload dependency lock file
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./pipeline/seq_retrieval/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            pipeline/seq_retrieval/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "pipeline/seq_retrieval/pyproject.toml"
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Upload updated (main) lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: pipeline_seq_retrieval_deps_lock
          path: pipeline/seq_retrieval/requirements.txt
          if-no-files-found: error
      - name: Upload updated (test) lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: pipeline_seq_retrieval_tests_deps_lock
          path: pipeline/seq_retrieval/tests/requirements.txt
          if-no-files-found: error
  api-update-dependency-lock-files:
    name: api update and upload dependency lock file
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./api/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            api/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "api/pyproject.toml"
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Upload updated (main) lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: api_deps_lock
          path: api/requirements.txt
          if-no-files-found: error
      - name: Upload updated (test) lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: api_tests_deps_lock
          path: api/tests/requirements.txt
          if-no-files-found: error
  webui-update-dependency-lock-files:
    name: webui update and upload dependency lock file
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./webui/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            webui/
      - uses: actions/setup-node@v4
        with:
          node-version-file: "webui/.nvmrc"
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-lock
      - name: Upload updated lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: webui_deps_lock
          path: webui/package-lock.json
          if-no-files-found: error
  shared-aws-py-package-update-dependency-lock-files:
    name: shared_aws py-package update and upload dependency lock file
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./shared_aws/py_package/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            shared_aws/py_package/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "shared_aws/py_package/pyproject.toml"
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Upload updated (main) lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: shared_aws_py_package_deps_lock
          path: shared_aws/py_package/requirements.txt
          if-no-files-found: error
      - name: Upload updated (test) lock file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: shared_aws_py_package_test_deps_lock
          path: shared_aws/py_package/tests/requirements.txt
          if-no-files-found: error
  shared-aws-py-package-code-checks:
    name: shared_aws py-package code checks
    needs:
      - shared-aws-py-package-update-dependency-lock-files
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: ./shared_aws/py_package/
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        sparse-checkout: |
          Makefile
          shared_aws/py_package/
    - name: Download updated shared_aws py-package prod dependencies lock file
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package_deps_lock
        path: shared_aws/py_package
    - name: Download updated shared_aws py-package test dependencies lock file
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package_test_deps_lock
        path: shared_aws/py_package/tests
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "shared_aws/py_package/pyproject.toml"
    - name: Code typing test
      run: |
        make run-type-checks
    - name: Code style test
      run: |
        make run-style-checks
    - name: Unit tests
      run: make run-unit-tests
  shared-aws-py-package-build:
    name: shared_aws python package build
    needs:
      - shared-aws-py-package-code-checks
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: ./shared_aws/py_package/
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        sparse-checkout: |
          Makefile
          shared_aws/py_package/
    - name: Download updated shared_aws py-package prod dependencies lock file
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package_deps_lock
        path: shared_aws/py_package
    - name: Download updated shared_aws py-package test dependencies lock file
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package_test_deps_lock
        path: shared_aws/py_package/tests
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "shared_aws/py_package/pyproject.toml"
    - name: Build package
      run: |
        make clean build
    - name: Upload package as artifact
      uses: actions/upload-artifact@v4
      with:
        name: shared_aws_py_package
        path: shared_aws/py_package/dist/pavi_shared_aws-0.0.0-py3-none-any.whl
  shared-aws-infra-update-dependency-lock-files:
    name: shared_aws aws_infra update and upload dependency lock file
    needs:
      - shared-aws-py-package-build
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./shared_aws/aws_infra/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            shared_aws/aws_infra/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "shared_aws/aws_infra/pyproject.toml"
      - name: Download shared AWS package
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_py_package
          path: /tmp/
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Bundle all deps lock files
        run: |
          tar -czv -f shared-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
      - name: Upload deps-lock-files bundle as artifact
        uses: actions/upload-artifact@v4
        with:
          name: shared_aws_infra_deps_locks_bundle
          path: shared_aws/aws_infra/shared-aws-infra-deps-locks.tar.gz
          if-no-files-found: error
  shared-aws-infra-code-checks:
    name: shared_aws infra code checks
    needs:
      - shared-aws-infra-update-dependency-lock-files
      - shared-aws-py-package-build
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: ./shared_aws/aws_infra/
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        sparse-checkout: |
          Makefile
          shared_aws/aws_infra/
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: Download updated shared_aws/aws_infra dependencies lock files bundle
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_infra_deps_locks_bundle
        path: shared_aws/aws_infra
    - name: Unpack the bundle
      run: |
        tar -xzv -f shared-aws-infra-deps-locks.tar.gz
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "shared_aws/aws_infra/pyproject.toml"
    - name: Code typing test
      run: |
        make run-type-checks
    - name: Code style test
      run: |
        make run-style-checks
    - name: Unit tests
      run: make run-unit-tests
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "shared_aws/aws_infra/.nvmrc"
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
        aws-region: us-east-1
    - name: Validate production CDK stack code
      run: make validate-stack
  pipeline-aws-infra-update-dependency-lock-files:
    name: pipeline/aws_infra update and upload dependency lock file
    needs:
      - shared-aws-py-package-build
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./pipeline/aws_infra/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            pipeline/aws_infra/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "pipeline/aws_infra/pyproject.toml"
      - name: Download shared AWS package
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_py_package
          path: /tmp/
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Bundle all deps lock files
        run: |
          tar -czv -f pipeline-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
      - name: Upload deps-lock-files bundle as artifact
        uses: actions/upload-artifact@v4
        with:
          name: pipeline_aws_infra_deps_locks_bundle
          path: pipeline/aws_infra/pipeline-aws-infra-deps-locks.tar.gz
          if-no-files-found: error
  api-aws-infra-update-dependency-lock-files:
    name: api/aws_infra update and upload dependency lock file
    needs:
      - shared-aws-py-package-build
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./api/aws_infra/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            api/aws_infra/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "api/aws_infra/pyproject.toml"
      - name: Download shared AWS package
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_py_package
          path: /tmp/
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Bundle all deps lock files
        run: |
          tar -czv -f api-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
      - name: Upload deps-lock-files bundle as artifact
        uses: actions/upload-artifact@v4
        with:
          name: api_aws_infra_deps_locks_bundle
          path: api/aws_infra/api-aws-infra-deps-locks.tar.gz
          if-no-files-found: error
  webui-aws-infra-update-dependency-lock-files:
    name: webui/aws_infra update and upload dependency lock file
    needs:
      - shared-aws-py-package-build
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./webui/aws_infra/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            webui/aws_infra/
      - uses: actions/setup-python@v5
        with:
          python-version-file: "webui/aws_infra/pyproject.toml"
      - name: Download shared AWS package
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_py_package
          path: /tmp/
      - name: Update lock files
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
        shell: bash
        run: |
          make update-deps-locks-all
      - name: Bundle all deps lock files
        run: |
          tar -czv -f webui-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
      - name: Upload deps-lock-files bundle as artifact
        uses: actions/upload-artifact@v4
        with:
          name: webui_aws_infra_deps_locks_bundle
          path: webui/aws_infra/webui-aws-infra-deps-locks.tar.gz
          if-no-files-found: error
  pipeline-seq-retrieval-container-image-build:
    name: pipeline/seq_retrieval container-image build
    needs:
      - container-build-context
      - pipeline-seq-retrieval-update-dependency-lock-files
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            pipeline/seq_retrieval/
      - name: Download updated seq_retrieval (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: pipeline_seq_retrieval_deps_lock
          path: pipeline/seq_retrieval
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      # This step will configure environment variables to be used by all steps
      # involving AWS interaction further down
      - name: AWS credentials configuration
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-seq-retrieval-image-upload
          aws-region: us-east-1
      - name: Amazon ECR login
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2
      - name: Build container image
        uses: docker/build-push-action@v6
        env:
          DOCKER_BUILD_SUMMARY: false
        with:
          context: ./pipeline/seq_retrieval/
          push: false
          tags: agr_pavi/pipeline_seq_retrieval:latest
          outputs: type=docker,dest=/tmp/pavi_seq_retrieval_docker_image.tar
      - name: Upload image as artifact (share between jobs)
        uses: actions/upload-artifact@v4
        with:
          name: seq_retrieval_image
          path: /tmp/pavi_seq_retrieval_docker_image.tar
      - name: Load, tag and push image to registry
        run: |
          docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
          docker tag agr_pavi/pipeline_seq_retrieval:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
          docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
  pipeline-seq-retrieval-code-checks:
    name: pipeline/seq_retrieval code checks
    needs:
      - pipeline-seq-retrieval-update-dependency-lock-files
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./pipeline/seq_retrieval/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            pipeline/seq_retrieval/
      - name: Download updated seq_retrieval (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: pipeline_seq_retrieval_deps_lock
          path: pipeline/seq_retrieval
      - name: Download updated seq_retrieval (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: pipeline_seq_retrieval_tests_deps_lock
          path: pipeline/seq_retrieval/tests
      - uses: actions/setup-python@v5
        with:
          python-version-file: "pipeline/seq_retrieval/pyproject.toml"
      - name: Code typing test
        run: |
          make run-type-checks
      - name: Code style test
        run: |
          make run-style-checks
      - name: Run unit tests
        run: |
          make run-unit-tests
  pipeline-alignment-container-image-build:
    name: pipeline/alignment container-image build
    needs:
      - container-build-context
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            pipeline/alignment/
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      # This step will configure environment variables to be used by all steps
      # involving AWS interaction further down
      - name: AWS credentials configuration
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-alignment-image-upload
          aws-region: us-east-1
      - name: Amazon ECR login
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2
      - name: Build container image
        uses: docker/build-push-action@v6
        env:
          DOCKER_BUILD_SUMMARY: false
        with:
          context: ./pipeline/alignment/
          push: false
          tags: agr_pavi/pipeline_alignment:latest
          outputs: type=docker,dest=/tmp/pavi_alignment_docker_image.tar
      - name: Upload image as artifact (share between jobs)
        uses: actions/upload-artifact@v4
        with:
          name: alignment_image
          path: /tmp/pavi_alignment_docker_image.tar
      - name: Load, tag and push image to registry
        run: |
          docker load --input /tmp/pavi_alignment_docker_image.tar
          docker tag agr_pavi/pipeline_alignment:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
          docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
  pipeline-alignment-unit-testing:
    name: pipeline/alignment unit testing
    needs:
      - pipeline-alignment-container-image-build
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./pipeline/alignment/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            pipeline/alignment/
      - name: Download alignment image artifact (from previous job)
        uses: actions/download-artifact@v4
        with:
          name: alignment_image
          path: /tmp
      - name: Load alignment Docker image
        run: |
          docker load --input /tmp/pavi_alignment_docker_image.tar
      - name: Run unit test
        run: |
          make run-unit-tests
  pipeline-workflow-integration-testing:
    name: pipeline/workflow integration testing
    needs:
      - container-build-context
      - pipeline-seq-retrieval-container-image-build
      - pipeline-alignment-container-image-build
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./pipeline/workflow/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            pipeline/workflow/
            tests/resources/
      - name: Download seq_retrieval image artifact (from previous job)
        uses: actions/download-artifact@v4
        with:
          name: seq_retrieval_image
          path: /tmp
      - name: Download alignment image artifact (from previous job)
        uses: actions/download-artifact@v4
        with:
          name: alignment_image
          path: /tmp
      - name: Load seq_retrieval Docker image
        run: |
          docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
      - name: Load alignment Docker image
        run: |
          docker load --input /tmp/pavi_alignment_docker_image.tar
      - name: Run local integration test
        run: |
          make run-integration-test-local
      # This step will configure environment variables to be used by all steps
      # involving AWS interaction further down
      - name: AWS credentials configuration
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-workflow-aws-test
          aws-region: us-east-1
      - name: Run AWS integration test
        run: |
          make run-integration-test-aws TAG_NAME=${{needs.container-build-context.outputs.tagname}}
  pipeline-aws-infra-code-checks:
    name: pipeline/aws_infra code checks
    needs:
      - pipeline-aws-infra-update-dependency-lock-files
      - shared-aws-py-package-build
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: ./pipeline/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        sparse-checkout: |
          Makefile
          pipeline/aws_infra/
    - name: Download updated pipeline/aws_infra dependencies lock files bundle
      uses: actions/download-artifact@v4
      with:
        name: pipeline_aws_infra_deps_locks_bundle
        path: pipeline/aws_infra
    - name: Unpack the bundle
      working-directory: pipeline/aws_infra/
      run: |
        tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "pipeline/aws_infra/pyproject.toml"
    - name: Code typing test
      run: |
        make run-type-checks
    - name: Code style test
      run: |
        make run-style-checks
    - name: Unit tests
      run: make run-unit-tests
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "pipeline/aws_infra/.nvmrc"
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
        aws-region: us-east-1
    - name: Validate production CDK stack code
      run: make validate-stack
  api-code-checks:
    name: API code checks
    needs:
      - api-update-dependency-lock-files
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./api/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            api/
      - name: Download updated api (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_deps_lock
          path: api
      - name: Download updated api (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_tests_deps_lock
          path: api/tests
      - uses: actions/setup-python@v5
        with:
          python-version-file: "api/pyproject.toml"
      - name: Code typing test
        run: |
          make run-type-checks
      - name: Code style test
        run: |
          make run-style-checks
  api-container-image-build:
    name: API container-image build
    needs:
      - api-update-dependency-lock-files
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Download updated api (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_deps_lock
          path: api
      - name: Download updated api (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_tests_deps_lock
          path: api/tests
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      # Configure environment variables to be pass on to container build
      # (require to mount S3 volume before nextflow plugin download)
      - name: AWS credentials configuration
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-api-aws-test
          aws-region: us-east-1
      - name: Store AWS credentials in (AWS CLI compatible) config and credentials files
        run: |
          echo "[default]" > aws.config
          echo "region = $AWS_REGION" >> aws.config
          echo "[default]" > aws.credentials
          echo "aws_access_key_id = $AWS_ACCESS_KEY_ID" >> aws.credentials
          echo "aws_secret_access_key = $AWS_SECRET_ACCESS_KEY" >> aws.credentials
          echo "aws_session_token = $AWS_SESSION_TOKEN" >> aws.credentials
      - name: Build container image
        uses: docker/build-push-action@v6
        env:
          DOCKER_BUILD_SUMMARY: false
        with:
          context: ./
          file: api/Dockerfile
          push: false
          secrets: |
            AWS_PROFILE=default
          secret-files: |
            AWS_SHARED_CREDENTIALS_FILE=aws.credentials
            AWS_CONFIG_FILE=aws.config
          tags: agr_pavi/api:latest
          outputs: type=docker,dest=/tmp/pavi_api_docker_image.tar
      - name: Upload image as artifact (share between jobs)
        uses: actions/upload-artifact@v4
        with:
          name: api_image
          path: /tmp/pavi_api_docker_image.tar
  api-unit-integration-testing:
    name: API unit and integration testing
    needs:
      - api-update-dependency-lock-files
      - pipeline-seq-retrieval-container-image-build
      - pipeline-alignment-container-image-build
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./api/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Download updated api (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_deps_lock
          path: api
      - name: Download updated api (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_tests_deps_lock
          path: api/tests
      - name: Download seq_retrieval image artifact (from previous job)
        uses: actions/download-artifact@v4
        with:
          name: seq_retrieval_image
          path: /tmp
      - name: Download alignment image artifact (from previous job)
        uses: actions/download-artifact@v4
        with:
          name: alignment_image
          path: /tmp
      - name: Load seq_retrieval Docker image
        run: |
          docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
      - name: Load alignment Docker image
        run: |
          docker load --input /tmp/pavi_alignment_docker_image.tar
      - uses: actions/setup-python@v5
        with:
          python-version-file: "api/pyproject.toml"
      - name: Run unit and integration tests
        run: |
          make run-tests
  api-container-integration-testing:
    name: API container integration testing (AWS batch execution)
    needs:
      - container-build-context
      - pipeline-seq-retrieval-container-image-build
      - pipeline-alignment-container-image-build
      - api-container-image-build
      - api-update-dependency-lock-files
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./api/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            api/
            tests/resources/
      - name: Download updated api (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_deps_lock
          path: api
      - name: Download updated api (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_tests_deps_lock
          path: api/tests
      - name: Download API image artifact (from previous job)
        uses: actions/download-artifact@v4
        with:
          name: api_image
          path: /tmp
      - name: Load API Docker image
        run: |
          docker load --input /tmp/pavi_api_docker_image.tar
      - uses: actions/setup-python@v5
        with:
          python-version-file: "api/pyproject.toml"
      # This step will configure environment variables to be used by all steps
      # involving AWS interaction further down
      - name: AWS credentials configuration
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-api-aws-test
          aws-region: us-east-1
      - name: Run container integration test (AWS execution)
        run: |
          make run-integration-test-container TAG_NAME=${{needs.container-build-context.outputs.tagname}}
  api-aws-infra-code-checks:
    name: api/aws_infra code checks
    needs:
      - api-aws-infra-update-dependency-lock-files
      - shared-aws-py-package-build
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: ./api/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        sparse-checkout: |
          Makefile
          api/aws_infra/
    - name: Download updated api/aws_infra dependencies lock files bundle
      uses: actions/download-artifact@v4
      with:
        name: api_aws_infra_deps_locks_bundle
        path: api/aws_infra
    - name: Unpack the bundle
      working-directory: api/aws_infra/
      run: |
        tar -xzv -f api-aws-infra-deps-locks.tar.gz
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "api/aws_infra/pyproject.toml"
    - name: Code typing test
      run: |
        make run-type-checks
    - name: Code style test
      run: |
        make run-style-checks
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "api/aws_infra/.nvmrc"
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
        aws-region: us-east-1
    - name: Validate production CDK stack code
      run: make validate-all
  webui-aws-infra-code-checks:
    name: webui/aws_infra code checks
    needs:
      - webui-aws-infra-update-dependency-lock-files
      - shared-aws-py-package-build
    permissions:
      id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: ./webui/aws_infra
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        sparse-checkout: |
          Makefile
          webui/aws_infra/
    - name: Download updated webui/aws_infra dependencies lock files bundle
      uses: actions/download-artifact@v4
      with:
        name: webui_aws_infra_deps_locks_bundle
        path: webui/aws_infra
    - name: Unpack the bundle
      working-directory: webui/aws_infra/
      run: |
        tar -xzv -f webui-aws-infra-deps-locks.tar.gz
    - name: Download shared AWS package
      uses: actions/download-artifact@v4
      with:
        name: shared_aws_py_package
        path: /tmp/
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version-file: "webui/aws_infra/pyproject.toml"
    - name: Code typing test
      run: |
        make run-type-checks
    - name: Code style test
      run: |
        make run-style-checks
    - name: Setup node.js (CDK CLI requirement)
      uses: actions/setup-node@v4
      with:
        node-version-file: "webui/aws_infra/.nvmrc"
    - name: AWS credentials configuration
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
        role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
        aws-region: us-east-1
    - name: Validate production CDK stack code
      run: make validate-all
  webui-code-checks:
    name: webUI code checks
    needs:
      - webui-update-dependency-lock-files
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: ./webui/
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          sparse-checkout: |
            Makefile
            webui/
      - name: Download updated webui dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: webui_deps_lock
          path: webui
      - uses: actions/setup-node@v4
        with:
          node-version-file: "webui/.nvmrc"
      - name: Code typing test
        run: |
          make run-type-checks
      - name: Code style test
        run: |
          make run-style-checks
      - name: Unit tests
        run: make run-unit-tests
  webui-container-image-build:
    name: webUI container-image build
    needs:
      - webui-update-dependency-lock-files
    runs-on: ubuntu-22.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Download updated webui dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: webui_deps_lock
          path: webui
      - name: Report node version stored in .nvmrc
        id: nvmrc-node-version
        run: |
          {
            echo 'content<<EOF'
            cat webui/.nvmrc
            echo EOF
          } >> "$GITHUB_OUTPUT"
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Build container image
        uses: docker/build-push-action@v6
        env:
          DOCKER_BUILD_SUMMARY: false
        with:
          context: ./webui/
          file: webui/Dockerfile
          push: false
          build-args: NODE_VERSION=${{steps.nvmrc-node-version.outputs.content}}
          tags: agr_pavi/webui:latest
          outputs: type=docker,dest=/tmp/pavi_webui_docker_image.tar
      - name: Upload image as artifact (share between jobs)
        uses: actions/upload-artifact@v4
        with:
          name: webui_image
          path: /tmp/pavi_webui_docker_image.tar
  e2e-testing:
    name: end-to-end testing
    needs:
      - container-build-context
      - webui-update-dependency-lock-files
      - webui-container-image-build
      - api-container-image-build
      - pipeline-alignment-container-image-build
      - pipeline-seq-retrieval-container-image-build
    permissions:
      id-token: write # Required for OIDC authentication to AWS (to perform AWS actions / access AGR AWS resources)
    runs-on: ubuntu-22.04
    steps:
      # Configure environment variables used for AWS interaction,
      # required for API container interactions with pipeline resources
      - name: AWS credentials configuration
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-e2e-testing
          aws-region: us-east-1
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Download API image artifact
        uses: actions/download-artifact@v4
        with:
          name: api_image
          path: /tmp
      - name: Load API Docker image
        run: |
          docker load --input /tmp/pavi_api_docker_image.tar
      - name: Run local API container to run E2E tests on
        working-directory: api/
        run: |
          API_PIPELINE_IMAGE_TAG=${{needs.container-build-context.outputs.tagname}} make run-container-dev
      - name: Download webUI image artifact
        uses: actions/download-artifact@v4
        with:
          name: webui_image
          path: /tmp
      - name: Load webUI Docker image
        run: |
          docker load --input /tmp/pavi_webui_docker_image.tar
      - name: Run local webUI container to run E2E tests on
        working-directory: webui/
        run: |
          PAVI_API_PORT=8080 make run-container-dev
      - name: Download updated webui dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: webui_deps_lock
          path: webui
      - name: setup webui-compatible node.js version
        uses: actions/setup-node@v4
        with:
          node-version-file: "webui/.nvmrc"
      - name: Run cypress E2E tests
        uses: cypress-io/github-action@v6
        with:
          working-directory: webui/
          wait-on: 'http://localhost:8080/api/health, http://localhost:3000/health'
          env: 'API_BASE_URL=http://localhost:8080'
      - name: Report API container logs on E2E test failure
        if: failure()
        run: |
          docker logs agr.pavi.dev-local.api.server
      - name: Report webUI container logs on E2E test failure
        if: failure()
        run: |
          docker logs agr.pavi.dev-local.webui.server
      - name: Cleanup webUI server (running container)
        if: success() || failure()
        working-directory: webui/
        run: |
          make stop-container-dev
      - name: Cleanup API server (running container)
        if: success() || failure()
        working-directory: api/
        run: |
          make stop-container-dev
  stage-deps-lock-updates:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write
    # Always stage updated lock files, even on failure,
    #  to highlight dependency changes which could be the cause of the failure.
    needs:
      - shared-aws-py-package-update-dependency-lock-files
      - shared-aws-infra-update-dependency-lock-files
      - api-aws-infra-update-dependency-lock-files
      - pipeline-aws-infra-update-dependency-lock-files
      - webui-aws-infra-update-dependency-lock-files
      - api-update-dependency-lock-files
      - pipeline-seq-retrieval-update-dependency-lock-files
      - webui-update-dependency-lock-files
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          fetch-depth: 0
      - name: Download updated seq_retrieval (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: pipeline_seq_retrieval_deps_lock
          path: pipeline/seq_retrieval
      - name: Download updated seq_retrieval (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: pipeline_seq_retrieval_tests_deps_lock
          path: pipeline/seq_retrieval/tests
      - name: Download updated api (main) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_deps_lock
          path: api
      - name: Download updated api (test) dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: api_tests_deps_lock
          path: api/tests
      - name: Download updated pipeline/aws_infra dependencies lock files bundle
        uses: actions/download-artifact@v4
        with:
          name: pipeline_aws_infra_deps_locks_bundle
          path: pipeline/aws_infra
      - name: Unpack the bundle
        working-directory: pipeline/aws_infra/
        run: |
          tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz
      - name: Download updated shared_aws py-package prod dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_py_package_deps_lock
          path: shared_aws/py_package
      - name: Download updated shared_aws py-package test dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_py_package_test_deps_lock
          path: shared_aws/py_package/tests
      - name: Download updated shared_aws/aws_infra dependencies lock files bundle
        uses: actions/download-artifact@v4
        with:
          name: shared_aws_infra_deps_locks_bundle
          path: shared_aws/aws_infra
      - name: Unpack the bundle
        working-directory: shared_aws/aws_infra/
        run: |
          tar -xzv -f shared-aws-infra-deps-locks.tar.gz
      - name: Download updated api/aws_infra dependencies lock files bundle
        uses: actions/download-artifact@v4
        with:
          name: api_aws_infra_deps_locks_bundle
          path: api/aws_infra
      - name: Unpack the bundle
        working-directory: api/aws_infra/
        run: |
          tar -xzv -f api-aws-infra-deps-locks.tar.gz
      - name: Download updated webui/aws_infra dependencies lock files bundle
        uses: actions/download-artifact@v4
        with:
          name: webui_aws_infra_deps_locks_bundle
          path: webui/aws_infra
      - name: Unpack the bundle
        working-directory: webui/aws_infra/
        run: |
          tar -xzv -f webui-aws-infra-deps-locks.tar.gz
      - name: Download updated webui dependencies lock file
        uses: actions/download-artifact@v4
        with:
          name: webui_deps_lock
          path: webui
      - name: Bundle all deps lock files
        run: |
          find . -regex '.*/\(requirements\.txt\|package-lock\.json\)' -print | \
          tar -czv -f deps-lock-files.tar.gz --files-from -
      # Upload the deps-lock-files bundle as workflow artifact (to be downloaded on PR merge)
      - name: Upload deps-lock-files bundle as artifact
        uses: actions/upload-artifact@v4
        with:
          name: deps_lock_files_bundle
          path: deps-lock-files.tar.gz
      # Share diff of changes in PR
      - name: Generate list of files that were updated
        id: diff-files-list
        run: |
            git diff --name-only -- > diff-files.txt
            {
              echo 'content<<EOF'
              cat diff-files.txt
              echo EOF
            } >> "$GITHUB_OUTPUT"
      - name: Calculate diff for each individual file
        id: diff-files-content
        if: ${{ steps.diff-files-list.outputs.content }}
        run: |
          mkdir diff-files/
          cat diff-files.txt | xargs -I {} bash ci-scripts/create-git-diff-markdown.sh {} diff-files
          {
            echo 'total_char_count<<EOF'
            find ./diff-files/ -type f -exec cat {} \; | wc -m
            echo EOF
          } >> "$GITHUB_OUTPUT"
      - name: Generate and store change message summary
        if: ${{ steps.diff-files-list.outputs.content }}
        run: |
          {
            echo 'The validation workflow automatically applied the below dependency lock file updates.'
            echo 'A bundle with all dependency lock files is available [here](${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}) as artifact called `deps_lock_files_bundle`.'
          } > git-diff-message.md
      # gh pr addComment API has a max 65536 character limit
      - name: Report all diffs in a single comment
        if: ${{ steps.diff-files-content.outputs.total_char_count < 60000 }}
        run: |
          {
            echo ''
            cat diff-files.txt | xargs -I {} cat diff-files/{}.md
          } >> git-diff-message.md
      - name: Generate and store no-change message
        if: ${{ !steps.diff-files-list.outputs.content }}
        run: |
          echo 'Automatic dependency lock file updates enabled, but no updates found. No (additional) dependency updates will be made when merging this PR.' \
           > git-diff-message.md
      - name: Post PR comment with change message
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file git-diff-message.md
      - name: Report each file diff in a separate comment
        if: ${{ steps.diff-files-content.outputs.total_char_count >= 60000 }}
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          cat diff-files.txt | xargs -I {} gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file diff-files/{}.md
  report-no-deps-lock-updates-label:
    if: ${{ contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write
    steps:
      - name: Post PR comment with notification no updates will be made
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh pr comment ${{ github.event.pull_request._links.html.href }} --body '`no-deps-lock-updates` label detected, so automatic dependency lock file updates are disabled. No (additional) dependency updates will be made when merging this PR.'