build: on windows (GreptimeTeam#2054)

* build on windows

* rebase develop

* fix: resolve PR comments

.config/nextest.toml

@@ -1,2 +1,3 @@
 [profile.default]
 slow-timeout = { period = "60s", terminate-after = 3, grace-period = "30s" }
+retries = { backoff = "exponential", count = 3, delay = "10s", jitter = true }

.github/workflows/develop.yml

@@ -70,52 +70,13 @@ jobs:
       - name: Run taplo
         run: taplo format --check
 
-  # Use coverage to run test.
-  # test:
-  #   name: Test Suite
-  #   if: github.event.pull_request.draft == false
-  #   runs-on: ubuntu-latest
-  #   timeout-minutes: 60
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: Cache LLVM and Clang
-  #       id: cache-llvm
-  #       uses: actions/cache@v3
-  #       with:
-  #         path: ./llvm
-  #         key: llvm
-  #     - uses: arduino/setup-protoc@v1
-  #       with:
-  #         repo-token: ${{ secrets.GITHUB_TOKEN }}
-  #     - uses: KyleMayes/install-llvm-action@v1
-  #       with:
-  #         version: "14.0"
-  #         cached: ${{ steps.cache-llvm.outputs.cache-hit }}
-  #     - uses: dtolnay/rust-toolchain@master
-  #       with:
-  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
-  #     - name: Rust Cache
-  #       uses: Swatinem/rust-cache@v2
-  #     - name: Cleanup disk
-  #       uses: curoky/[email protected]
-  #       with:
-  #         retain: 'rust,llvm'
-  #     - name: Install latest nextest release
-  #       uses: taiki-e/install-action@nextest
-  #     - name: Run tests
-  #       run: cargo nextest run
-  #       env:
-  #         CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
-  #         RUST_BACKTRACE: 1
-  #         GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-  #         GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-  #         GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-  #         UNITTEST_LOG_DIR: "__unittest_logs"
-
   sqlness:
     name: Sqlness Test
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest-8-cores, windows-latest ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -127,25 +88,14 @@ jobs:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-      - name: Run etcd
-        run: |
-          ETCD_VER=v3.5.7
-          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
-          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download
-          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
-          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-
-          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
-          nohup etcd >/tmp/etcd.log 2>&1 &
       - name: Run sqlness
-        run: cargo sqlness && ls /tmp
+        run: cargo sqlness
       - name: Upload sqlness logs
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: sqlness-logs
-          path: /tmp/greptime-*.log
+          path: ${{ runner.temp }}/greptime-*.log
           retention-days: 3
 
   fmt:
@@ -234,3 +184,43 @@ jobs:
           flags: rust
           fail_ci_if_error: false
           verbose: true
+
+  test-on-windows:
+    if: github.event.pull_request.draft == false
+    runs-on: windows-latest-8-cores
+    timeout-minutes: 60
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+          distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"

Cargo.toml

@@ -18,7 +18,6 @@ members = [
     "src/common/meta",
     "src/common/procedure",
     "src/common/procedure-test",
-    "src/common/pprof",
     "src/common/query",
     "src/common/recordbatch",
     "src/common/runtime",

scripts/fetch-dashboard-assets.sh

@@ -2,14 +2,14 @@
 
 # This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
 
-set -e
+set -e -x
 
 declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
 declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
 declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
 OUT_DIR="${1:-$SCRIPT_DIR}"
 
-RELEASE_VERSION="$(cat $STATIC_DIR/VERSION)"
+RELEASE_VERSION="$(cat $STATIC_DIR/VERSION | tr -d '\t\r\n ')"
 
 echo "Downloading assets to dir: $OUT_DIR"
 cd $OUT_DIR

src/cmd/Cargo.toml

@@ -54,15 +54,19 @@ session = { workspace = true }
 snafu.workspace = true
 substrait = { workspace = true }
 table = { workspace = true }
-tikv-jemallocator = "0.5"
 tokio.workspace = true
 
+[target.'cfg(not(windows))'.dependencies]
+tikv-jemallocator = "0.5"
+
 [dev-dependencies]
 common-test-util = { workspace = true }
-rexpect = "0.5"
 serde.workspace = true
 temp-env = "0.3"
 toml.workspace = true
 
+[target.'cfg(not(windows))'.dev-dependencies]
+rexpect = "0.5"
+
 [build-dependencies]
 common-version = { workspace = true }

src/cmd/src/bin/greptime.rs

@@ -187,6 +187,7 @@ fn log_env_flags() {
     }
 }
 
+#[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;