rust-cubesql.yml

name: Build native

on:
  push:
    paths:
      - '.github/workflows/rust-cubesql.yml'
      - 'packages/cubejs-backend-native/**'
      - 'rust/cubenativeutils/**'
      - 'rust/cubesqlplanner/**'
      - 'rust/cubesql/**'
    branches:
      - 'master'
  pull_request:
    paths:
      - '.github/workflows/rust-cubesql.yml'
      - 'packages/cubejs-backend-native/**'
      - 'rust/cubenativeutils/**'
      - 'rust/cubesqlplanner/**'
      - 'rust/cubesql/**'

jobs:
  lint:
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    name: Check fmt/clippy
    container:
      image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2024-07-15
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt, clippy
          cache: false
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ./rust/cubesql -> target
          # default key
          key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu
          shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu
      - name: Lint CubeSQL
        run: cd rust/cubesql && cargo fmt --all -- --check
      - name: Lint Native
        run: cd packages/cubejs-backend-native && cargo fmt --all -- --check
      - name: Lint cubenativeutils
        run: cd rust/cubenativeutils && cargo fmt --all -- --check
      - name: Lint cubesqlplanner
        run: cd rust/cubesqlplanner && cargo fmt --all -- --check
      - name: Clippy CubeSQL
        run: cd rust/cubesql && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
      - name: Clippy Native
        run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
      - name: Clippy Native (with Python)
        run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going --features python -- -D warnings

  unit:
    #  We use host instead of cross container, because it's much faster
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    name: Unit (Rewrite Engine) (CUBESQL_TOP_DOWN_EXTRACTOR=${{ matrix.top-down-extractor }})
    strategy:
      matrix:
        top-down-extractor: ['true', 'false']
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # pulls all commits (needed for codecov)
          fetch-depth: 2
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ./rust/cubesql -> target
          # default key
          key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu
          shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu
      - name: Install lld
        run: |
          # ubuntu 24.04 has lld-18
          sudo apt-get update && sudo apt install -y lld
      - name: Install cargo-llvm-cov@0.6.10
        uses: baptiste0928/cargo-install@v3
        with:
          crate: cargo-llvm-cov
          version: "0.6.10"
      - name: Install cargo-insta@1.42.0
        uses: baptiste0928/cargo-install@v3
        with:
          crate: cargo-insta
          version: "1.42.0"
      - name: Unit tests (Rewrite Engine)
        env:
          CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }}
          CUBESQL_TESTING_CUBE_URL: ${{ secrets.CUBESQL_TESTING_CUBE_URL }}
          CUBESQL_SQL_PUSH_DOWN: true
          CUBESQL_TOP_DOWN_EXTRACTOR: ${{ matrix.top-down-extractor }}
          CUBESQL_REWRITE_CACHE: true
          CUBESQL_REWRITE_TIMEOUT: 60
        run: |
          cd rust/cubesql
          # See https://github.com/taiki-e/cargo-llvm-cov/blob/main/README.md#get-coverage-of-external-tests
          # shellcheck source=/dev/null
          source <(cargo llvm-cov show-env --export-prefix)
          cargo insta test --all-features --workspace --unreferenced reject
          cargo llvm-cov report --workspace --lcov --output-path lcov.info
      - name: Upload code coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./rust/cubesql/lcov.info
          verbose: true
          flags: cubesql
          fail_ci_if_error: false

  native_linux:
    needs: [lint]
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    name: Build Linux GNU ${{ matrix.node-version }}.x ${{ matrix.target }} with Python ${{ matrix.python-version }}
    strategy:
      matrix:
        # Current used version + 1 LTS
        node-version: [20, 22]
        python-version: ["3.9", "3.10", "3.11", "3.12", "fallback"]
        target: ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]
        # minimize number of jobs
        exclude:
          - node-version: 20
            target: "aarch64-unknown-linux-gnu"
          - python-version: 3.10
            target: "aarch64-unknown-linux-gnu"
          - python-version: 3.11
            target: "aarch64-unknown-linux-gnu"
      fail-fast: false
    container:
      image: cubejs/rust-cross:${{ matrix.target }}-15082024${{ matrix.python-version != 'fallback' && format('-python-{0}', matrix.python-version) || '' }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2024-07-15
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
          target: ${{ matrix.target }}
          cache: false
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ./rust/cubesql -> target
          key: cubesql-${{ matrix.target }}
          shared-key: cubesql-${{ matrix.target }}
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install Yarn
        run: npm install -g yarn
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Build native (fallback)
        if: (matrix.python-version == 'fallback')
        env:
          CARGO_BUILD_TARGET: ${{ matrix.target }}
        working-directory: ./packages/cubejs-backend-native
        run: yarn run native:build-debug
      - name: Setup cross compilation
        if: (matrix.target == 'aarch64-unknown-linux-gnu')
        uses: allenevans/set-env@v4.0.0
        with:
          PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }}
      - name: Build native (with Python)
        if: (matrix.python-version != 'fallback')
        env:
          PYO3_PYTHON: python${{ matrix.python-version }}
          CARGO_BUILD_TARGET: ${{ matrix.target }}
        working-directory: ./packages/cubejs-backend-native
        run: yarn run native:build-debug-python
      - name: Test native (GNU only)
        if: (matrix.target == 'x86_64-unknown-linux-gnu')
        env:
          CUBESQL_STREAM_MODE: true
          CUBEJS_NATIVE_INTERNAL_DEBUG: true
        working-directory: ./packages/cubejs-backend-native
        run: yarn run test:unit
      - name: Run E2E Smoke testing over whole Cube (GNU only)
        if: (matrix.target == 'x86_64-unknown-linux-gnu')
        env:
          CUBEJS_NATIVE_INTERNAL_DEBUG: true
        working-directory: ./packages/cubejs-testing
        run: yarn smoke:cubesql

  native_macos:
    needs: [lint]
    runs-on: ${{ matrix.os-version }}
    timeout-minutes: 60
    name: Build ${{ matrix.os-version }} ${{ matrix.target }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }}

    strategy:
      matrix:
        # We do not need to test under all versions, we do it under linux
        node-version: [20.x]
        os-version: ["macos-13"]
        target: ["x86_64-apple-darwin", "aarch64-apple-darwin"]
        include:
          - target: x86_64-apple-darwin
            python-version: "3.9"
          - target: x86_64-apple-darwin
            python-version: "3.10"
          - target: x86_64-apple-darwin
            python-version: "3.11"
          - target: x86_64-apple-darwin
            python-version: "3.12"
          - target: x86_64-apple-darwin
            python-version: "fallback"
          - target: aarch64-apple-darwin
            python-version: "fallback"
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2024-07-15
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
          target: ${{ matrix.target }}
      - name: Install Python
        uses: actions/setup-python@v5
        if: (matrix.python-version != 'fallback')
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Build native (fallback)
        if: (matrix.python-version == 'fallback')
        env:
          CUBESQL_STREAM_MODE: true
          CUBEJS_NATIVE_INTERNAL_DEBUG: true
          CARGO_BUILD_TARGET: ${{ matrix.target }}
        run: cd packages/cubejs-backend-native && yarn run native:build
      - name: Build native (with Python)
        if: (matrix.python-version != 'fallback')
        env:
          PYO3_PYTHON: python${{ matrix.python-version }}
          CARGO_BUILD_TARGET: ${{ matrix.target }}
        run: cd packages/cubejs-backend-native && yarn run native:build-debug-python
      - name: Tests
        # We cannot test arm64 on x64
        if: (matrix.target == 'x86_64-apple-darwin')
        env:
          CUBESQL_STREAM_MODE: true
          CUBEJS_NATIVE_INTERNAL_DEBUG: true
        run: cd packages/cubejs-backend-native && yarn run test:unit

  native_windows:
    needs: [lint]
    runs-on: ${{ matrix.os-version }}
    timeout-minutes: 60
    name: Build ${{ matrix.os-version }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }}

    strategy:
      matrix:
        # We do not need to test under all versions, we do it under linux
        node-version: [20.x]
        os-version: [windows-2019]
        python-version: ["fallback"]
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Disable rustup update (issue workaround for Windows)
        run: rustup set auto-self-update disable
        shell: bash
      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: nightly-2024-07-15
          # override: true # this is by default on
          rustflags: ""
          components: rustfmt
      - name: Install Python
        uses: actions/setup-python@v5
        if: (matrix.python-version != 'fallback')
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Set Yarn version
        run: yarn policies set-version v1.22.22
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Restore yarn cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Yarn install
        uses: nick-fields/retry@v3
        env:
          CUBESTORE_SKIP_POST_INSTALL: true
        with:
          max_attempts: 3
          retry_on: error
          retry_wait_seconds: 15
          timeout_minutes: 20
          command: yarn install --frozen-lockfile
      - name: Lerna tsc
        run: yarn tsc
      - name: Build native (fallback)
        if: (matrix.python-version == 'fallback')
        env:
          CUBESQL_STREAM_MODE: true
          CUBEJS_NATIVE_INTERNAL_DEBUG: true
        run: cd packages/cubejs-backend-native && yarn run native:build
      - name: Build native (with Python)
        if: (matrix.python-version != 'fallback')
        env:
          PYO3_PYTHON: python${{ matrix.python-version }}
        run: cd packages/cubejs-backend-native && yarn run native:build-debug-python
      - name: Tests
        env:
          CUBESQL_STREAM_MODE: true
          CUBEJS_NATIVE_INTERNAL_DEBUG: true
        run: cd packages/cubejs-backend-native && yarn run test:unit