From 1420c3ab657ac36a77a2654409934d669995d6c4 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sat, 8 Jun 2024 22:25:31 -0700 Subject: [PATCH 1/6] separate java builds --- .github/workflows/java_11.yml | 55 +++++++++++++++++++++ .github/workflows/{java.yml => java_17.yml} | 31 ++---------- .github/workflows/java_8.yml | 49 ++++++++++++++++++ 3 files changed, 107 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/java_11.yml rename .github/workflows/{java.yml => java_17.yml} (72%) create mode 100644 .github/workflows/java_8.yml diff --git a/.github/workflows/java_11.yml b/.github/workflows/java_11.yml new file mode 100644 index 0000000000..e0c870acc9 --- /dev/null +++ b/.github/workflows/java_11.yml @@ -0,0 +1,55 @@ +name: Build and Run Java 11 JNI Tests +on: + push: + branches: + - main + pull_request: + paths: + - java/** + - rust/** + - .github/workflows/java_11.yml +env: + # This env var is used by Swatinem/rust-cache@v2 for the cache + # key, so we set it to make sure it is always consistent. + CARGO_TERM_COLOR: always + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + RUST_BACKTRACE: "1" + # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html + # CI builds are faster with incremental disabled. + CARGO_INCREMENTAL: "0" + CARGO_BUILD_JOBS: "1" +jobs: + linux-build: + runs-on: ubuntu-22.04 + name: ubuntu-22.04 + Java 11 + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: java/java-jni + - name: Run cargo fmt + run: cargo fmt --check + working-directory: ./java/core/lance-jni + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + cache: "maven" + - name: Java Style Check + run: mvn checkstyle:check + - name: Rust Clippy + working-directory: java/core/lance-jni + run: cargo clippy --all-targets -- -D warnings + - name: Running tests with Java 11 + run: mvn clean test \ No newline at end of file diff --git a/.github/workflows/java.yml b/.github/workflows/java_17.yml similarity index 72% rename from .github/workflows/java.yml rename to .github/workflows/java_17.yml index a7a533fd8a..fd0b66e95e 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java_17.yml @@ -1,4 +1,4 @@ -name: Build and Run Java JNI Tests +name: Build and Run Java 17 JNI Tests on: push: branches: @@ -7,7 +7,7 @@ on: paths: - java/** - rust/** - - .github/workflows/java.yml + - .github/workflows/java_17.yml env: # This env var is used by Swatinem/rust-cache@v2 for the cache # key, so we set it to make sure it is always consistent. @@ -23,7 +23,7 @@ env: jobs: linux-build: runs-on: ubuntu-22.04 - name: ubuntu-22.04 + Java 11 & 17 + name: ubuntu-22.04 + Java 17 defaults: run: working-directory: ./java @@ -33,9 +33,6 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: java/java-jni - - name: Run cargo fmt - run: cargo fmt --check - working-directory: ./java/core/lance-jni - name: Install dependencies run: | sudo apt update @@ -47,28 +44,6 @@ jobs: java-version: 17 cache: "maven" - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV - - name: Install Java 8 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - cache: "maven" - - run: echo "JAVA_8=$JAVA_HOME" >> $GITHUB_ENV - - name: Install Java 11 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - cache: "maven" - - name: Java Style Check - run: mvn checkstyle:check - - name: Rust Clippy - working-directory: java/core/lance-jni - run: cargo clippy --all-targets -- -D warnings - - name: Running tests with Java 11 - run: mvn clean test - - name: Running tests with Java 8 - run: JAVA_HOME=$JAVA_8 mvn clean test - name: Running tests with Java 17 run: | export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \ diff --git a/.github/workflows/java_8.yml b/.github/workflows/java_8.yml new file mode 100644 index 0000000000..7a4e8c3040 --- /dev/null +++ b/.github/workflows/java_8.yml @@ -0,0 +1,49 @@ +name: Build and Run Java 8 JNI Tests +on: + push: + branches: + - main + pull_request: + paths: + - java/** + - rust/** + - .github/workflows/java_8.yml +env: + # This env var is used by Swatinem/rust-cache@v2 for the cache + # key, so we set it to make sure it is always consistent. + CARGO_TERM_COLOR: always + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + RUST_BACKTRACE: "1" + # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html + # CI builds are faster with incremental disabled. + CARGO_INCREMENTAL: "0" + CARGO_BUILD_JOBS: "1" +jobs: + linux-build: + runs-on: ubuntu-22.04 + name: ubuntu-22.04 + Java 8 + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: java/java-jni + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Install Java 8 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + cache: "maven" + - run: echo "JAVA_8=$JAVA_HOME" >> $GITHUB_ENV + - name: Running tests with Java 8 + run: JAVA_HOME=$JAVA_8 mvn clean test + \ No newline at end of file From 7730220a60b59f6a09f612c9a05ffd2bece1b3a4 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 11 Jun 2024 08:46:15 -0700 Subject: [PATCH 2/6] Combine into one yml file --- .github/workflows/{java_17.yml => java.yml} | 70 +++++++++++++++++++-- .github/workflows/java_11.yml | 55 ---------------- .github/workflows/java_8.yml | 49 --------------- 3 files changed, 65 insertions(+), 109 deletions(-) rename .github/workflows/{java_17.yml => java.yml} (57%) delete mode 100644 .github/workflows/java_11.yml delete mode 100644 .github/workflows/java_8.yml diff --git a/.github/workflows/java_17.yml b/.github/workflows/java.yml similarity index 57% rename from .github/workflows/java_17.yml rename to .github/workflows/java.yml index fd0b66e95e..1962552095 100644 --- a/.github/workflows/java_17.yml +++ b/.github/workflows/java.yml @@ -1,4 +1,5 @@ -name: Build and Run Java 17 JNI Tests +name: Build and Run Java JNI Tests + on: push: branches: @@ -7,7 +8,8 @@ on: paths: - java/** - rust/** - - .github/workflows/java_17.yml + - .github/workflows/java_jni_tests.yml + env: # This env var is used by Swatinem/rust-cache@v2 for the cache # key, so we set it to make sure it is always consistent. @@ -20,8 +22,67 @@ env: # CI builds are faster with incremental disabled. CARGO_INCREMENTAL: "0" CARGO_BUILD_JOBS: "1" + jobs: - linux-build: + build-and-test-java-11: + runs-on: ubuntu-22.04 + name: ubuntu-22.04 + Java 11 + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: java/java-jni + - name: Run cargo fmt + run: cargo fmt --check + working-directory: ./java/core/lance-jni + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + cache: "maven" + - name: Java Style Check + run: mvn checkstyle:check + - name: Rust Clippy + working-directory: java/core/lance-jni + run: cargo clippy --all-targets -- -D warnings + - name: Running tests with Java 11 + run: mvn clean test + + build-and-test-java-8: + runs-on: ubuntu-22.04 + name: ubuntu-22.04 + Java 8 + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: java/java-jni + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Install Java 8 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + cache: "maven" + - name: Running tests with Java 8 + run: mvn clean test + + build-and-test-java-17: runs-on: ubuntu-22.04 name: ubuntu-22.04 + Java 17 defaults: @@ -43,7 +104,6 @@ jobs: distribution: temurin java-version: 17 cache: "maven" - - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV - name: Running tests with Java 17 run: | export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \ @@ -65,4 +125,4 @@ jobs: --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ -Djdk.reflect.useDirectMethodHandle=false \ -Dio.netty.tryReflectionSetAccessible=true" - JAVA_HOME=$JAVA_17 mvn clean test + mvn clean test diff --git a/.github/workflows/java_11.yml b/.github/workflows/java_11.yml deleted file mode 100644 index e0c870acc9..0000000000 --- a/.github/workflows/java_11.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build and Run Java 11 JNI Tests -on: - push: - branches: - - main - pull_request: - paths: - - java/** - - rust/** - - .github/workflows/java_11.yml -env: - # This env var is used by Swatinem/rust-cache@v2 for the cache - # key, so we set it to make sure it is always consistent. - CARGO_TERM_COLOR: always - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - RUST_BACKTRACE: "1" - # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html - # CI builds are faster with incremental disabled. - CARGO_INCREMENTAL: "0" - CARGO_BUILD_JOBS: "1" -jobs: - linux-build: - runs-on: ubuntu-22.04 - name: ubuntu-22.04 + Java 11 - defaults: - run: - working-directory: ./java - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: java/java-jni - - name: Run cargo fmt - run: cargo fmt --check - working-directory: ./java/core/lance-jni - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Install Java 11 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - cache: "maven" - - name: Java Style Check - run: mvn checkstyle:check - - name: Rust Clippy - working-directory: java/core/lance-jni - run: cargo clippy --all-targets -- -D warnings - - name: Running tests with Java 11 - run: mvn clean test \ No newline at end of file diff --git a/.github/workflows/java_8.yml b/.github/workflows/java_8.yml deleted file mode 100644 index 7a4e8c3040..0000000000 --- a/.github/workflows/java_8.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Build and Run Java 8 JNI Tests -on: - push: - branches: - - main - pull_request: - paths: - - java/** - - rust/** - - .github/workflows/java_8.yml -env: - # This env var is used by Swatinem/rust-cache@v2 for the cache - # key, so we set it to make sure it is always consistent. - CARGO_TERM_COLOR: always - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - RUST_BACKTRACE: "1" - # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html - # CI builds are faster with incremental disabled. - CARGO_INCREMENTAL: "0" - CARGO_BUILD_JOBS: "1" -jobs: - linux-build: - runs-on: ubuntu-22.04 - name: ubuntu-22.04 + Java 8 - defaults: - run: - working-directory: ./java - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: java/java-jni - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Install Java 8 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - cache: "maven" - - run: echo "JAVA_8=$JAVA_HOME" >> $GITHUB_ENV - - name: Running tests with Java 8 - run: JAVA_HOME=$JAVA_8 mvn clean test - \ No newline at end of file From d47b3f3e5f6c7d927e60ddb709959c8665af564e Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 11 Jun 2024 08:47:39 -0700 Subject: [PATCH 3/6] Small fix to trigger PR build --- java/core/src/main/java/com/lancedb/lance/Dataset.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/core/src/main/java/com/lancedb/lance/Dataset.java b/java/core/src/main/java/com/lancedb/lance/Dataset.java index e59998ef26..4e322f2164 100644 --- a/java/core/src/main/java/com/lancedb/lance/Dataset.java +++ b/java/core/src/main/java/com/lancedb/lance/Dataset.java @@ -217,7 +217,7 @@ public long latestVersion() { /** * Count the number of rows in the dataset. * - * @return num of rows. + * @return num of rows */ public int countRows() { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { From f2d3b88d83d947c41eccdc94b1529c91043bcd63 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 11 Jun 2024 20:49:30 -0700 Subject: [PATCH 4/6] Use matrix --- .github/workflows/java.yml | 105 +++++++++------------------- java/core/lance-jni/src/fragment.rs | 7 +- 2 files changed, 37 insertions(+), 75 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 1962552095..9e80f0bc80 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -8,7 +8,7 @@ on: paths: - java/** - rust/** - - .github/workflows/java_jni_tests.yml + - .github/workflows/java.yml env: # This env var is used by Swatinem/rust-cache@v2 for the cache @@ -24,67 +24,28 @@ env: CARGO_BUILD_JOBS: "1" jobs: - build-and-test-java-11: + rust-clippy-fmt: runs-on: ubuntu-22.04 - name: ubuntu-22.04 + Java 11 + name: Rust Clippy and Fmt Check defaults: run: - working-directory: ./java + working-directory: ./java/core/lance-jni steps: - name: Checkout repository uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - workspaces: java/java-jni + workspaces: java/core/lance-jni - name: Run cargo fmt run: cargo fmt --check - working-directory: ./java/core/lance-jni - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Install Java 11 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - cache: "maven" - - name: Java Style Check - run: mvn checkstyle:check - name: Rust Clippy - working-directory: java/core/lance-jni run: cargo clippy --all-targets -- -D warnings - - name: Running tests with Java 11 - run: mvn clean test - - build-and-test-java-8: - runs-on: ubuntu-22.04 - name: ubuntu-22.04 + Java 8 - defaults: - run: - working-directory: ./java - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: java/java-jni - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Install Java 8 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - cache: "maven" - - name: Running tests with Java 8 - run: mvn clean test - build-and-test-java-17: + build-and-test: runs-on: ubuntu-22.04 - name: ubuntu-22.04 + Java 17 + strategy: + matrix: + java-version: [8, 11, 17] defaults: run: working-directory: ./java @@ -93,36 +54,38 @@ jobs: uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - workspaces: java/java-jni + workspaces: java/core/lance-jni - name: Install dependencies run: | sudo apt update sudo apt install -y protobuf-compiler libssl-dev - - name: Install Java 17 + - name: Set up Java ${{ matrix.java-version }} uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17 + java-version: ${{ matrix.java-version }} cache: "maven" - - name: Running tests with Java 17 + - name: Running tests with Java ${{ matrix.java-version }} run: | - export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \ - -XX:+IgnoreUnrecognizedVMOptions \ - --add-opens=java.base/java.lang=ALL-UNNAMED \ - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ - --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ - --add-opens=java.base/java.io=ALL-UNNAMED \ - --add-opens=java.base/java.net=ALL-UNNAMED \ - --add-opens=java.base/java.nio=ALL-UNNAMED \ - --add-opens=java.base/java.util=ALL-UNNAMED \ - --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ - --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ - --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ - --add-opens=java.base/sun.security.action=ALL-UNNAMED \ - --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ - --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ - -Djdk.reflect.useDirectMethodHandle=false \ - -Dio.netty.tryReflectionSetAccessible=true" + if [ "${{ matrix.java-version }}" == "17" ]; then + export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \ + -XX:+IgnoreUnrecognizedVMOptions \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/java.util=ALL-UNNAMED \ + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ + --add-opens=java.base/sun.security.action=ALL-UNNAMED \ + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ + -Djdk.reflect.useDirectMethodHandle=false \ + -Dio.netty.tryReflectionSetAccessible=true" + fi mvn clean test diff --git a/java/core/lance-jni/src/fragment.rs b/java/core/lance-jni/src/fragment.rs index 24d9863b22..2265822c11 100644 --- a/java/core/lance-jni/src/fragment.rs +++ b/java/core/lance-jni/src/fragment.rs @@ -34,10 +34,6 @@ use crate::{ RT, }; -/////////////////// -// Write Methods // -/////////////////// - ////////////////// // Read Methods // ////////////////// @@ -70,6 +66,9 @@ fn inner_count_rows_native( Ok(res) } +/////////////////// +// Write Methods // +/////////////////// #[no_mangle] pub extern "system" fn Java_com_lancedb_lance_Fragment_createWithFfiArray<'local>( mut env: JNIEnv<'local>, From e48840242d8b05d64622550602fec52977ebcb8f Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 11 Jun 2024 20:51:29 -0700 Subject: [PATCH 5/6] Small name fix --- .github/workflows/java.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 9e80f0bc80..26d2e707d0 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -41,11 +41,12 @@ jobs: - name: Rust Clippy run: cargo clippy --all-targets -- -D warnings - build-and-test: + build-and-test-java: runs-on: ubuntu-22.04 strategy: matrix: java-version: [8, 11, 17] + name: Build and Test with Java ${{ matrix.java-version }} defaults: run: working-directory: ./java From c49df9b829bf7c031e33759560a3701b55133469 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 11 Jun 2024 20:59:24 -0700 Subject: [PATCH 6/6] fix build --- .github/workflows/java.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 26d2e707d0..5c245ed313 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -36,6 +36,10 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: java/core/lance-jni + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev - name: Run cargo fmt run: cargo fmt --check - name: Rust Clippy