python_build.yml

name: python_build

on:
  merge_group:
  push:
    branches: [main]
  pull_request:
    branches: [main]

defaults:
  run:
    working-directory: ./python

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env

      - name: Check Python
        run: |
          uv pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions --system
          make check-python

      - name: Check Rust
        run: make check-rust

  test-minimal:
    name: Python Build (Python 3.9 PyArrow 16.0.0)
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=line-tables-only"
      CARGO_INCREMENTAL: 0

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env
        with:
          python-version: 3.9

      - name: Build and install deltalake
        run: |
          # Install minimum PyArrow version
          uv sync --extra devel --extra pandas --extra polars
          uv pip install pyarrow==16.0.0
        env:
          RUSTFLAGS: "-C debuginfo=line-tables-only"

      - name: Run tests
        run: make unit-test

  test:
    name: Python Build (Python 3.10 PyArrow latest)
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=1"
      CARGO_INCREMENTAL: 0

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env

      - name: Start emulated services
        run: docker compose up -d

      - name: Build and install deltalake
        run: make develop

      - name: Download Data Acceptance Tests (DAT) files
        run: make setup-dat

      - name: Run tests
        run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules

      - name: Test without pandas
        run: |
          uv pip uninstall pandas
          uv run --no-sync pytest -m "not pandas and not integration and not benchmark"
          uv pip install pandas

  test-lakefs:
    name: Python Build (Python 3.10 LakeFS Integration tests)
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=1"
      CARGO_INCREMENTAL: 0

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env

      - name: Start emulated services
        run: docker compose -f ../docker-compose-lakefs.yml up -d

      - name: Build and install deltalake
        run: make develop

      - name: Download Data Acceptance Tests (DAT) files
        run: make setup-dat

      - name: Run tests
        run: uv run --no-sync pytest -m '(lakefs and integration)' --doctest-modules

  test-unitycatalog-databricks:
    name: Python Build (Python 3.10 Unity Catalog Integration tests)
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=1"
      CARGO_INCREMENTAL: 0

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env

      - name: Run Mockoon CLI
        uses: mockoon/cli-action@v2
        with:
          version: "latest"
          data-file: ".github/mockoon_data_files/unitycatalog_databricks.json"
          port: 8080

      - name: Build and install deltalake
        run: make develop

      - name: Download Data Acceptance Tests (DAT) files
        run: make setup-dat

      - name: Run tests
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 10
          max_attempts: 3
          retry_on: error
          command: uv run --no-sync pytest -m '(unitycatalog_databricks and integration)' --doctest-modules

  test-unitycatalog-oss:
    name: Python Build (Python 3.10 Unity Catalog Integration tests)
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=1"
      CARGO_INCREMENTAL: 0

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env

      - name: Run Mockoon CLI
        uses: mockoon/cli-action@v2
        with:
          version: "latest"
          data-file: ".github/mockoon_data_files/unitycatalog_oss.json"
          port: 8080

      - name: Build and install deltalake
        run: make develop

      - name: Download Data Acceptance Tests (DAT) files
        run: make setup-dat

      - name: Run tests
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 10
          max_attempts: 3
          retry_on: error
          command: uv run --no-sync pytest -m '(unitycatalog_oss and integration)' --doctest-modules

  test-pyspark:
    name: PySpark Integration Tests
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=line-tables-only"
      CARGO_INCREMENTAL: 0

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env

      - uses: actions/setup-java@v2
        with:
          distribution: "zulu"
          java-version: "11"

      - name: Build and install deltalake
        run: make develop-pyspark

      - name: Run tests
        run: make test-pyspark


  multi-python-running:
    name: Running with Python ${{ matrix.python-version }}
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-C debuginfo=line-tables-only"
      CARGO_INCREMENTAL: 0

    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v3

      - name: Setup Environment
        uses: ./.github/actions/setup-env
        with:
          python-version: ${{ matrix.python-version }}

      - name: Build and install deltalake
        run: |
          uv sync

      - name: Run deltalake
        run: |
          uv run python -c 'import deltalake'