From 571593e1999b7acdbf4efbd19e5323d5955bb64e Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 9 Jul 2024 22:45:16 +0800 Subject: [PATCH 01/27] Add back jacoco github CI action. --- .github/workflows/build.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cbb9eaffb68..c48629b1237 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -107,3 +107,21 @@ jobs: build/reports catalogs/**/*.log catalogs/**/*.tar + + - name: Jacoco Report to PR + id: jacoco + uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 + with: + paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: 40 + min-coverage-changed-files: 60 + title: 'Code Coverage Report' + debug-mode: false + update-comment: true + pass-emoji: ':green_circle:' + fail-emoji: ':red_circle:' + - name: Get the Coverage info + run: | + echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}" From da85fad6e37889e875d2912fe59e419417498770 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 9 Jul 2024 22:47:23 +0800 Subject: [PATCH 02/27] changes --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c48629b1237..503123f3933 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 + uses: madrapps/jacoco-report@v1.6.1 with: paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} From 09251bc8197a05711da76082adeaedb2cd17080d Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 9 Jul 2024 23:00:55 +0800 Subject: [PATCH 03/27] Revert --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 503123f3933..c48629b1237 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.6.1 + uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 with: paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} From 9d3091e7228a77a104f76f0f5c4c3bf4fe89b581 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 10 Jul 2024 09:26:15 +0800 Subject: [PATCH 04/27] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c48629b1237..b2b88063f70 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -104,7 +104,7 @@ jobs: with: name: unit test report path: | - build/reports + ${{ github.workspace }}/**/build/reports catalogs/**/*.log catalogs/**/*.tar From 7491f9791acf98b1923b4c2e1a286d239ba0c46e Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 10 Jul 2024 09:39:48 +0800 Subject: [PATCH 05/27] fix --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2b88063f70..35e0c969787 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,7 +112,11 @@ jobs: id: jacoco uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 with: - paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml + paths: | + ${{ github.workspace }}/api/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/core/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 min-coverage-changed-files: 60 From 2dd61e883097f389776bfdc55aa7a2b214790934 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 10 Jul 2024 09:53:51 +0800 Subject: [PATCH 06/27] Fix --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35e0c969787..4578911ea7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,9 +113,9 @@ jobs: uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 with: paths: | - ${{ github.workspace }}/api/build/reports/jacoco/test/jacocoTestReport.xml - ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml - ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/api/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, ${{ github.workspace }}/core/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 From e2d020a5a89568417eca69106a9915d2004886d1 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 10 Jul 2024 10:24:26 +0800 Subject: [PATCH 07/27] Fix --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4578911ea7a..6a0db02afd6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,6 +76,10 @@ jobs: java-version: [ 8, 11, 17 ] timeout-minutes: 30 needs: changes + permissions: + contents: write + actions: write + if: needs.changes.outputs.source_changes == 'true' # Steps represent a sequence of tasks that will be executed as part of the job steps: From f1f8906d9fd9b87d754fda8c8e0a758a25117595 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 10 Jul 2024 10:41:12 +0800 Subject: [PATCH 08/27] Fix --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6a0db02afd6..11ad9d600c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,10 +76,7 @@ jobs: java-version: [ 8, 11, 17 ] timeout-minutes: 30 needs: changes - permissions: - contents: write - actions: write - + permissions: write-all if: needs.changes.outputs.source_changes == 'true' # Steps represent a sequence of tasks that will be executed as part of the job steps: From fc8c4cf3097e5cdfa3c0cd6815a73f2c4bfd039c Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 17:35:24 +0800 Subject: [PATCH 09/27] Fix --- .../workflows/backend-integration-test.yml | 384 +++++++++--------- .github/workflows/build.yml | 7 +- .github/workflows/flink-integration-test.yml | 208 +++++----- .../workflows/frontend-integration-test.yml | 214 +++++----- .github/workflows/python-integration-test.yml | 172 ++++---- .github/workflows/spark-integration-test.yml | 220 +++++----- 6 files changed, 601 insertions(+), 604 deletions(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 1c6596f772a..cd84ad4cfa3 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -1,192 +1,192 @@ -name: Backend Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - integration-test/** - - integration-test-common/** - - iceberg/** - - meta/** - - server/** - - server-common/** - - trino-connector/** - - web/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - test-on-push: - needs: changes - if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - # Integration test for AMD64 architecture - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - test-mode: [ embedded, deploy ] - include: - - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' - backend: 'mysql' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log - - test-on-pr: - needs: changes - if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - # Integration test for AMD64 architecture - architecture: [ linux/amd64 ] - java-version: [ 17 ] - test-mode: [ embedded, deploy ] - include: - - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' - backend: 'mysql' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if: ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log \ No newline at end of file +#name: Backend Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - integration-test/** +# - integration-test-common/** +# - iceberg/** +# - meta/** +# - server/** +# - server-common/** +# - trino-connector/** +# - web/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# test-on-push: +# needs: changes +# if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') +# runs-on: ubuntu-latest +# timeout-minutes: 90 +# strategy: +# matrix: +# # Integration test for AMD64 architecture +# architecture: [linux/amd64] +# java-version: [ 8, 11, 17 ] +# test-mode: [ embedded, deploy ] +# include: +# - test-mode: 'embedded' +# backend: 'h2' +# - test-mode: 'deploy' +# backend: 'mysql' +# +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if : ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) +# id: integrationTest +# run: > +# ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false +# -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test +# -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test +# -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test +# -x :authorizations:authorization-ranger:test +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} +# path: | +# build/reports +# iceberg/iceberg-rest-server/build/*.log +# integration-test/build/*.log +# integration-test/build/*.tar +# integration-test/build/trino-ci-container-log +# distribution/package/logs/*.out +# distribution/package/logs/*.log +# catalogs/**/*.log +# catalogs/**/*.tar +# distribution/**/*.log +# +# test-on-pr: +# needs: changes +# if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') +# runs-on: ubuntu-latest +# timeout-minutes: 90 +# strategy: +# matrix: +# # Integration test for AMD64 architecture +# architecture: [ linux/amd64 ] +# java-version: [ 17 ] +# test-mode: [ embedded, deploy ] +# include: +# - test-mode: 'embedded' +# backend: 'h2' +# - test-mode: 'deploy' +# backend: 'mysql' +# +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if: ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) +# id: integrationTest +# run: > +# ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false +# -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test +# -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test +# -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test +# -x :authorizations:authorization-ranger:test +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} +# path: | +# build/reports +# iceberg/iceberg-rest-server/build/*.log +# integration-test/build/*.log +# integration-test/build/*.tar +# integration-test/build/trino-ci-container-log +# distribution/package/logs/*.out +# distribution/package/logs/*.log +# catalogs/**/*.log +# catalogs/**/*.tar +# distribution/**/*.log \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdcf427e76b..99e2e0b5758 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,13 +144,10 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 + uses: madrapps/jacoco-report@1.6 with: paths: | - ${{ github.workspace }}/api/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/core/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 min-coverage-changed-files: 60 diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml index 6e1227664cb..997dc7bb3de 100644 --- a/.github/workflows/flink-integration-test.yml +++ b/.github/workflows/flink-integration-test.yml @@ -1,104 +1,104 @@ -name: Flink Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - meta/** - - server/** - - server-common/** - - flink-connector/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Flink Integration Test - id: integrationTest - run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: flink-connector-integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - flink-connector/flink/build/*.log - flink-connector/flink/build/*.tar - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar +#name: Flink Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - meta/** +# - server/** +# - server-common/** +# - flink-connector/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 30 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8, 11, 17 ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Flink Integration Test +# id: integrationTest +# run: | +# ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" +# ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: flink-connector-integrate-test-reports-${{ matrix.java-version }} +# path: | +# build/reports +# flink-connector/flink/build/*.log +# flink-connector/flink/build/*.tar +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log +# catalogs/**/*.log +# catalogs/**/*.tar diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index 2d730f392ff..28a669230b3 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -1,107 +1,107 @@ -name: Frontend Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - integration-test/** - - meta/** - - server/** - - server-common/** - - spark-connector/** - - trino-connector/** - - web/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Frontend Integration Test - id: integrationTest - run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - integration-test/build/integration-test-integration-test.log - integration-test/build/*.tar - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file +#name: Frontend Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - integration-test/** +# - meta/** +# - server/** +# - server-common/** +# - spark-connector/** +# - trino-connector/** +# - web/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 60 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8 ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Frontend Integration Test +# id: integrationTest +# run: | +# ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" +# ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: integrate-test-reports-${{ matrix.java-version }} +# path: | +# build/reports +# integration-test/build/integration-test-integration-test.log +# integration-test/build/*.tar +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log +# catalogs/**/*.log +# catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 4e9f96bc690..abd85c901f1 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -1,86 +1,86 @@ -name: Python Client Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - catalogs/catalog-hadoop/** - - clients/client-python/** - - common/** - - conf/** - - core/** - - meta/** - - server/** - - server-common/** - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Python Client Integration Test - id: integrationTest - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - for pythonVersion in "3.8" "3.9" "3.10" "3.11" - do - echo "Use Python version ${pythonVersion} to test the Python client." - ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} -PskipDockerTests=false :clients:client-python:test - # Clean Gravitino database to clean test data - rm -rf ./distribution/package/data - done - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} - with: - name: integrate test reports - path: | - build/reports - integration-test/build/integration-test.log - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file +#name: Python Client Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - catalogs/catalog-hadoop/** +# - clients/client-python/** +# - common/** +# - conf/** +# - core/** +# - meta/** +# - server/** +# - server-common/** +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 30 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8 ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Python Client Integration Test +# id: integrationTest +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# for pythonVersion in "3.8" "3.9" "3.10" "3.11" +# do +# echo "Use Python version ${pythonVersion} to test the Python client." +# ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} -PskipDockerTests=false :clients:client-python:test +# # Clean Gravitino database to clean test data +# rm -rf ./distribution/package/data +# done +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} +# with: +# name: integrate test reports +# path: | +# build/reports +# integration-test/build/integration-test.log +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log +# catalogs/**/*.log +# catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml index a2f11061cda..b59261b2e10 100644 --- a/.github/workflows/spark-integration-test.yml +++ b/.github/workflows/spark-integration-test.yml @@ -1,110 +1,110 @@ -name: Spark Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - iceberg/** - - meta/** - - server/** - - server-common/** - - spark-connector/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - scala-version: [ 2.12 ] - test-mode: [ embedded, deploy ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Spark Integration Test - id: integrationTest - run: | - if [ "${{ matrix.scala-version }}" == "2.12" ];then - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - fi - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} - path: | - build/reports - spark-connector/v3.3/spark/build/spark-3.3-integration-test.log - spark-connector/v3.4/spark/build/spark-3.4-integration-test.log - spark-connector/v3.5/spark/build/spark-3.5-integration-test.log - distribution/package/logs/*.out - distribution/package/logs/*.log +#name: Spark Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - iceberg/** +# - meta/** +# - server/** +# - server-common/** +# - spark-connector/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 90 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8, 11, 17 ] +# scala-version: [ 2.12 ] +# test-mode: [ embedded, deploy ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if : ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Spark Integration Test +# id: integrationTest +# run: | +# if [ "${{ matrix.scala-version }}" == "2.12" ];then +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" +# fi +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} +# path: | +# build/reports +# spark-connector/v3.3/spark/build/spark-3.3-integration-test.log +# spark-connector/v3.4/spark/build/spark-3.4-integration-test.log +# spark-connector/v3.5/spark/build/spark-3.5-integration-test.log +# distribution/package/logs/*.out +# distribution/package/logs/*.log From 63db289b4f8b23e2cf9017a58469b30812d5779c Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 17:36:38 +0800 Subject: [PATCH 10/27] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99e2e0b5758..81c2b35c41b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,7 +144,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@1.6 + uses: madrapps/jacoco-report@v1.6 with: paths: | ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml From 993be77a78faa1070d7d716e3eed26f9c26aed78 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 17:56:45 +0800 Subject: [PATCH 11/27] Fix jacoco problem. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81c2b35c41b..5c627724bf9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,7 +144,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.6 + uses: madrapps/jacoco-report@v1.7 with: paths: | ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml From 8cd191b346226d7227379ea829d36850b0e497e3 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 18:02:28 +0800 Subject: [PATCH 12/27] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c627724bf9..119f4a30bb1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,7 +144,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.7 + uses: madrapps/jacoco-report@v1.7.0 with: paths: | ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml From 76555bc3fe5bbfccfd5ae01231489a7f3170fc16 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 19:24:50 +0800 Subject: [PATCH 13/27] Fix --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 119f4a30bb1..38e37f22894 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,7 +147,9 @@ jobs: uses: madrapps/jacoco-report@v1.7.0 with: paths: | - ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/server-common/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 min-coverage-changed-files: 60 From 3e74a7c2f040d0642e296b93b013518656cb240e Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 20:30:52 +0800 Subject: [PATCH 14/27] Fix --- .github/workflows/build.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38e37f22894..0d00c4a565b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,12 +144,9 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.7.0 + uses: madrapps/jacoco-report@v1.6.1 with: - paths: | - ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/server-common/build/reports/jacoco/test/jacocoTestReport.xml + paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 min-coverage-changed-files: 60 From bb9d1b894611a621fb5f7edfb0d088e02ab4eb4e Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 11:41:08 +0800 Subject: [PATCH 15/27] Test --- .github/workflows/build.yml | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0d00c4a565b..279eb61943b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,39 +68,6 @@ jobs: - name: Build with Gradle run: ./gradlew build -x test -PjdkVersion=8 - # To check the spark-connector is compatible with scala2.13 - spark-connector-build: - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: changes - if: needs.changes.outputs.source_changes == 'true' - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: 8 - distribution: 'temurin' - cache: 'gradle' - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Build with Scala2.13 - run: | - ./gradlew :spark-connector:spark-3.4:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false - ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false - - - name: Upload unit tests report - uses: actions/upload-artifact@v3 - if: failure() - with: - name: unit test report - path: | - build/reports - spark-connector/**/*.log - build: # The type of runner that the job will run on runs-on: ubuntu-latest @@ -122,8 +89,8 @@ jobs: distribution: 'temurin' cache: 'gradle' - - name: Test publish to local - run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }} +# - name: Test publish to local +# run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }} - name: Free up disk space run: | From 3531af777afdebacd0e74031f8bdef9487f46681 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 12:12:50 +0800 Subject: [PATCH 16/27] Trigger test. --- .../apache/gravitino/EntityStoreFactory.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java b/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java index 372e5faf4bb..1eb80daa944 100644 --- a/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java @@ -68,4 +68,21 @@ public static EntityStore createEntityStore(Config config) { throw new RuntimeException("Failed to create and initialize EntityStore: " + name, e); } } + + public static EntityStore createEntityStore(Config config) { + String name = config.get(Configs.ENTITY_STORE); + String className = ENTITY_STORES.getOrDefault(name, name); + + if (KV_STORE_KEY.equals(name)) { + throw new UnsupportedOperationException( + "KvEntityStore is not supported since version 0.6.0. Please use RelationalEntityStore instead."); + } + + try { + return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance(); + } catch (Exception e) { + LOG.error("Failed to create and initialize EntityStore by name {}.", name, e); + throw new RuntimeException("Failed to create and initialize EntityStore: " + name, e); + } + } } From b9e543217baf2e3d9bf890d5bdcdc120fd863775 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 12:14:29 +0800 Subject: [PATCH 17/27] Fix --- core/src/main/java/org/apache/gravitino/EntityStoreFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java b/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java index 1eb80daa944..1943b3c0be6 100644 --- a/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java @@ -69,7 +69,7 @@ public static EntityStore createEntityStore(Config config) { } } - public static EntityStore createEntityStore(Config config) { + public static EntityStore createEntityStore1(Config config) { String name = config.get(Configs.ENTITY_STORE); String className = ENTITY_STORES.getOrDefault(name, name); From 05266aad9817f4faa6c67a2bc200bc095c3d3002 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 15:05:27 +0800 Subject: [PATCH 18/27] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 279eb61943b..ae5b0ad5a78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,7 +73,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java-version: [ 8, 11, 17 ] + java-version: [ 8 ] timeout-minutes: 30 needs: changes permissions: write-all From 2bf194cb0d13b51329163335122f39152f8c873a Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 15:05:40 +0800 Subject: [PATCH 19/27] Fix --- .../access-control-integration-test.yml | 192 +++++++++--------- 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 9f7d5f2bdc4..7fed3ea1cda 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -1,96 +1,96 @@ -name: Access Control Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - api/** - - authorizations/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - common/** - - core/** - - integration-test-common/** - - server/** - - server-common/** - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - # Integration test for AMD64 architecture - architecture: [linux/amd64] - java-version: [ 17 ] - test-mode: [ embedded, deploy ] - include: - - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' - backend: 'mysql' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: | - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: authorizations-integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log +#name: Access Control Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - api/** +# - authorizations/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - common/** +# - core/** +# - integration-test-common/** +# - server/** +# - server-common/** +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 60 +# strategy: +# matrix: +# # Integration test for AMD64 architecture +# architecture: [linux/amd64] +# java-version: [ 17 ] +# test-mode: [ embedded, deploy ] +# include: +# - test-mode: 'embedded' +# backend: 'h2' +# - test-mode: 'deploy' +# backend: 'mysql' +# +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if : ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Authorization Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) +# id: integrationTest +# run: | +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdbcBackend=${{ matrix.backend }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: authorizations-integrate-test-reports-${{ matrix.java-version }} +# path: | +# build/reports +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log From f8f224ba49cee4105a29589e164dd783147b0d05 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 15:32:35 +0800 Subject: [PATCH 20/27] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae5b0ad5a78..a854eb7155b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,7 +111,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.6.1 + uses: madrapps/jacoco-report@v1.7.0 with: paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} From 250c7d7cf69f5cfdbd38bab462711c02586c4eba Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 16:03:02 +0800 Subject: [PATCH 21/27] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a854eb7155b..873bab229bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,7 +111,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.7.0 + uses: madrapps/jacoco-report@v1.5 with: paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} From c790099421eac865b2971d985667a1f58aa5d175 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 16:19:54 +0800 Subject: [PATCH 22/27] Fix --- .github/workflows/build.yml | 2 +- build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 873bab229bd..a854eb7155b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,7 +111,7 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@v1.5 + uses: madrapps/jacoco-report@v1.7.0 with: paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} diff --git a/build.gradle.kts b/build.gradle.kts index 3e47891f84f..39295739eb8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -534,7 +534,7 @@ tasks.cyclonedxBom { } jacoco { - toolVersion = "0.8.10" + toolVersion = "0.8.12" reportsDirectory.set(layout.buildDirectory.dir("JacocoReport")) } From 82b3d4bbe8544b60bb0e2a103396cdcb7ad4813f Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 16:43:10 +0800 Subject: [PATCH 23/27] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a854eb7155b..72cfcbf73b2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,7 +73,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java-version: [ 8 ] + java-version: [ 17 ] timeout-minutes: 30 needs: changes permissions: write-all From e48b799f66e9b0d4a9fd5b11ba07892370ed0919 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 16:56:31 +0800 Subject: [PATCH 24/27] Try to use codecov --- .github/workflows/build.yml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72cfcbf73b2..f3894b0072f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,20 +109,25 @@ jobs: catalogs/**/*.log catalogs/**/*.tar - - name: Jacoco Report to PR - id: jacoco - uses: madrapps/jacoco-report@v1.7.0 + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 with: - paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: 40 - min-coverage-changed-files: 60 - title: 'Code Coverage Report' - debug-mode: false - update-comment: true - pass-emoji: ':green_circle:' - fail-emoji: ':red_circle:' - - name: Get the Coverage info - run: | - echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}" + file: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml + +# - name: Jacoco Report to PR +# id: jacoco +# uses: madrapps/jacoco-report@v1.7.0 +# with: +# paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml +# token: ${{ secrets.GITHUB_TOKEN }} +# min-coverage-overall: 40 +# min-coverage-changed-files: 60 +# title: 'Code Coverage Report' +# debug-mode: false +# update-comment: true +# pass-emoji: ':green_circle:' +# fail-emoji: ':red_circle:' +# - name: Get the Coverage info +# run: | +# echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" +# echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}" From 22f393b313e3085e9cdf4a9d5a95408506d0f7fd Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 4 Sep 2024 17:14:58 +0800 Subject: [PATCH 25/27] FIx --- .github/workflows/build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3894b0072f..abfc90882ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,7 +112,14 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - file: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml + files: ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/clients/client-java/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/clients/client-java-runtime/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/clients/filesystem-hadoop3/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/clients/filesystem-hadoop3-runtime/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/spark-connector/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/flink-connector/build/reports/jacoco/test/jacocoTestReport.xml, + ${{ github.workspace }}/trino-connector/build/reports/jacoco/test/jacocoTestReport.xml # - name: Jacoco Report to PR # id: jacoco From 905d812c10884e6b113eb90c53bf935db83a3686 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 5 Sep 2024 20:30:54 +0800 Subject: [PATCH 26/27] Update. --- .github/workflows/build.yml | 6 +----- build.gradle.kts | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11ad9d600c8..0111ef1a0a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,11 +113,7 @@ jobs: id: jacoco uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 with: - paths: | - ${{ github.workspace }}/api/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/core/build/reports/jacoco/test/jacocoTestReport.xml + paths: ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 min-coverage-changed-files: 60 diff --git a/build.gradle.kts b/build.gradle.kts index e8d2b477ea0..0c27be66178 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -421,7 +421,10 @@ subprojects { jvmArgs = listOf("-Xmx2G") useJUnitPlatform() jvmArgs(project.property("extraJvmArgs") as List<*>) - finalizedBy(tasks.getByName("jacocoTestReport")) + + if (project.name != "integration-test-common") { + finalizedBy(tasks.getByName("jacocoTestReport")) + } } } From 3e302f791fdd06315e63d8eca9cd113abc93b922 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 5 Sep 2024 21:00:28 +0800 Subject: [PATCH 27/27] Fix --- .github/workflows/build.yml | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0a0d5706cfd..b1a61bc90eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,6 +68,39 @@ jobs: - name: Build with Gradle run: ./gradlew build -x test -PjdkVersion=8 + # To check the spark-connector is compatible with scala2.13 + spark-connector-build: + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: changes + if: needs.changes.outputs.source_changes == 'true' + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'temurin' + cache: 'gradle' + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Build with Scala2.13 + run: | + ./gradlew :spark-connector:spark-3.4:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false + ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false + + - name: Upload unit tests report + uses: actions/upload-artifact@v3 + if: failure() + with: + name: unit test report + path: | + build/reports + spark-connector/**/*.log + build: # The type of runner that the job will run on runs-on: ubuntu-latest @@ -75,8 +108,8 @@ jobs: matrix: java-version: [ 17 ] timeout-minutes: 30 - needs: changes permissions: write-all + needs: changes if: needs.changes.outputs.source_changes == 'true' # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -105,7 +138,7 @@ jobs: with: name: unit test report path: | - ${{ github.workspace }}/**/build/reports + build/reports catalogs/**/*.log catalogs/**/*.tar