test-push.yml

name: Test

permissions:
  id-token: write
  contents: read
  checks: write
  pull-requests: write


on:
  push:
    branches:
      - 'main'
      - 'release/**'
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - labeled
      - ready_for_review

concurrency:
  cancel-in-progress: true
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}

jobs:
  setup:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || (! github.event.pull_request.draft)
    outputs:
      matrix: ${{ steps.setup.outputs.matrix }}
    steps:
      - name: Setup tests matrix
        id: setup
        shell: python
        run: |
          import os
          import json
          
          matrix = {
              "os": [
                  "ubuntu-latest",
                  "windows-latest",
              ],
              "python-version": [
                  "3.11",
              ],
              "storage": [
                  "--hub-cloud --local",
                  "--s3 --gcs --azure --gdrive --memory-skip"
              ]
          }
          
          if "${{ github.event_name }}" == "pull_request":
              if "${{ contains(github.event.pull_request.labels.*.name, 'quick-tests') }}" == "true":
                  matrix = {
                      "os": [
                          "ubuntu-latest",
                      ],
                      "python-version": [
                          "3.11",
                      ],
                      "storage": [
                          "local",
                      ]
                  }
          
              if "${{ contains(github.event.pull_request.labels.*.name, 'macos-tests') }}" == "true":
                  matrix["os"].append("macos-latest")

              if "${{ contains(github.event.pull_request.labels.*.name, 'full-tests') }}" == "true":
                  matrix["os"] = [
                      "macos-latest",
                      "ubuntu-latest",
                      "windows-latest",
                  ]
                  matrix["python-version"] = [
                      "3.11",
                      "3.10",
                      "3.9",
                      "3.8",
                  ]
                  matrix["storage"] = [
                    "--hub-cloud --local",
                    "--s3 --gcs --azure --gdrive --memory-skip"
                  ]
          
          elif "${{ github.event_name }}" == "push":
              if "${{ github.ref_name == 'main' || startsWith( github.ref_name, 'release/' ) }}" == "true":
                  matrix["os"] = [
                      "macos-latest",
                      "ubuntu-latest",
                      "windows-latest",
                  ]
                  matrix["python-version"] = [
                      "3.11",
                      "3.10",
                      "3.9",
                      "3.8",
                  ]
              else:
                  raise Exception("Unexpected branch")
          else:
              raise Exception("Unexpected event")
          
          matrix["exclude"] = [
            {
              "os": "macos-latest",
              "python-version": "3.8",
            },
            {
              "os": "macos-latest",
              "python-version": "3.9",
            },
            {
              "os": "macos-latest",
              "python-version": "3.11",
            },
            {
              "os": "windows-latest",
              "python-version": "3.11",
            },
          ]
            
          with open(os.environ.get("GITHUB_OUTPUT"), "a") as file:
            file.write("matrix=%s\n" % json.dumps(matrix))

  test:
    name: Test
    needs: setup
    uses: activeloopai/shared-github-actions/.github/workflows/full_test.yml@main
    if: github.repository == 'activeloopai/deeplake'
    with:
      repo: ${{ github.repository }}
      ref: ${{ github.sha }}
      testMatrix: ${{ needs.setup.outputs.matrix }}

    secrets:
      token: ${{ secrets.GITHUB_TOKEN }}
      aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
      gcp_sa_credentials_json: ${{ secrets.GCP_SA_CREDENTIALS_JSON }}
      azure_creds_json: ${{ secrets.AZURE_CREDS_JSON }}
      hub_username: ${{ secrets.ACTIVELOOP_HUB_USERNAME }}
      hub_token: ${{ secrets.ACTIVELOOP_HUB_TOKEN }}
      kaggle_username: ${{ secrets.KAGGLE_USERNAME }}
      kaggle_key: ${{ secrets.KAGGLE_KEY }}
      oauth_client_id: ${{ secrets.GDRIVE_CLIENT_ID }}
      oauth_client_secret: ${{ secrets.GDRIVE_CLIENT_SECRET }}
      oauth_refresh_token: ${{ secrets.GDRIVE_REFRESH_TOKEN }}
      sonar_token: ${{ secrets.SONAR_TOKEN }}

  version-test:
    name: Check Python ${{ matrix.python-version }}
    needs: setup
    runs-on: ${{ matrix.os }}
    if: github.repository == 'activeloopai/deeplake' && !(github.ref_name == 'main' || startsWith( github.ref_name, 'release/' )) && !contains(github.event.pull_request.labels.*.name, 'full-tests')
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest"]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v3
        with:
          repository: ${{ inputs.repo }}
          ref: ${{ inputs.ref }}
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
          cache-dependency-path: deeplake/requirements/*.txt

      - name: Install requirements
        run: |
          pip3 install --upgrade pip --user
          pip3 install --upgrade setuptools
          pip3 install -r deeplake/requirements/common.txt
          pip3 install -r deeplake/requirements/tests.txt
          pip3 install -r deeplake/requirements/plugins.txt
          pip3 install -e .          

      - name: Run fast tests
        id: fast-tests
        timeout-minutes: 30
        run: |
          python -m pytest --local -m "not slow and not flaky" --timeout=60
        env:
          DEEPLAKE_PYTEST_ENABLED: true