FEAT-#4605: Implementation of Small Query Compiler to support small and empty DataFrames

.github/workflows/ci.yml

@@ -15,6 +15,22 @@ on:
       - setup.py
       - versioneer.py
 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      test-small-query-compiler: ${{ steps.filter.outputs.test-small-query-compiler }}
+    steps:
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          test-small-query-compiler:
+            - 'modin/experimental/core/storage_formats/pandas/small_query_compiler.py'
+            - 'modin/core/storage_formats/pandas/query_compiler.py'
+            - 'modin/core/storage_formats/base/query_compiler.py'
+
   lint-black:
     name: lint (black)
     runs-on: ubuntu-latest
@@ -969,3 +985,60 @@ jobs:
           conda info
           conda list
       - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py
+
+  test-small-query-compiler:
+    needs: [changes, lint-flake8, lint-black, test-api, test-headers]
+    if: ${{ needs.changes.outputs.test-small-query-compiler == 'true' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    env:
+      MODIN_SMALL_QUERY_COMPILER: "True"
+    name: test-small-query-compiler python ${{matrix.python-version}})
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Cache conda
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/.cache/pip
+          key:
+            ${{ runner.os }}-conda-${{ hashFiles('requirements/environment-py36.yml') }}
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          activate-environment: modin
+          environment-file: environment-dev.yml
+          python-version: 3.9
+          channel-priority: strict
+          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+      - name: Conda environment
+        run: |
+          conda info
+          conda list
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_iter.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_join_sort.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_map_metadata.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_pickle.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_reduce.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_udf.py
+      - run: pytest -n 2 modin/pandas/test/dataframe/test_window.py
+      - run: pytest -n 2 modin/pandas/test/test_api.py
+      - run: pytest -n 2 modin/pandas/test/test_concat.py
+      - run: pytest -n 2 modin/pandas/test/test_general.py
+      - run: pytest -n 2 modin/pandas/test/test_groupby.py
+      - run: pytest -n 2 modin/pandas/test/test_rolling.py
+      - run: pytest -n 2 modin/pandas/test/test_series.py
+      - uses: codecov/codecov-action@v2

modin/config/envvars.py

@@ -592,6 +592,13 @@ class ReadSqlEngine(EnvironmentVariable, type=str):
     choices = ("Pandas", "Connectorx")
 
 
+class InitializeWithSmallQueryCompilers(EnvironmentVariable, type=str):
+    """Set to true to use implementation of SmallQueryCompiler."""
+
+    varname = "MODIN_SMALL_QUERY_COMPILER"
+    default = False
+
+
 def _check_vars() -> None:
     """
     Check validity of environment variables.