From 486e85bc1178d8777adffba3a6ba5af8456cceae Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Tue, 11 Apr 2023 10:35:08 +0800 Subject: [PATCH 1/9] Run code style check first when run github ci actions. --- .github/workflows/core-ci.yml | 3 +++ .github/workflows/trino-ci.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index 04c9f279d1..b6a515a0fb 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -47,6 +47,9 @@ jobs: distribution: 'temurin' cache: maven + - name: Run code checkstyle + run: mvn validate + - name: Build all module with Maven run: mvn clean install -pl '!trino' diff --git a/.github/workflows/trino-ci.yml b/.github/workflows/trino-ci.yml index a50f46b960..24e6e53b41 100644 --- a/.github/workflows/trino-ci.yml +++ b/.github/workflows/trino-ci.yml @@ -46,6 +46,9 @@ jobs: distribution: 'temurin' cache: maven + - name: Run code checkstyle + run: mvn validate + - name: Install dependency with Maven run: mvn clean install -DskipTests -pl 'ams/ams-api,core,hive' -am From 1fedf01106353972e537b0569ca4a279cf0e57d0 Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Tue, 11 Apr 2023 10:42:45 +0800 Subject: [PATCH 2/9] skip checkstyle during ci build steps as of already run checkstyle before that. --- .github/workflows/core-ci.yml | 2 +- .github/workflows/trino-ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index b6a515a0fb..d346fdee8a 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -51,7 +51,7 @@ jobs: run: mvn validate - name: Build all module with Maven - run: mvn clean install -pl '!trino' + run: mvn clean install -pl '!trino' -Dcheckstyle.skip=true - name: Code coverage uses: codecov/codecov-action@v3 diff --git a/.github/workflows/trino-ci.yml b/.github/workflows/trino-ci.yml index 24e6e53b41..d6af2467c5 100644 --- a/.github/workflows/trino-ci.yml +++ b/.github/workflows/trino-ci.yml @@ -50,10 +50,10 @@ jobs: run: mvn validate - name: Install dependency with Maven - run: mvn clean install -DskipTests -pl 'ams/ams-api,core,hive' -am + run: mvn clean install -DskipTests -Dcheckstyle.skip=true -pl 'ams/ams-api,core,hive' -am - name: Build with Maven - run: mvn clean test -pl 'trino' + run: mvn clean test -pl 'trino' -Dcheckstyle.skip=true - name: Code coverage uses: codecov/codecov-action@v3 From e93b2dc3faa2530fda6055381e47ad1426302209 Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Tue, 11 Apr 2023 20:00:44 +0800 Subject: [PATCH 3/9] test ci for EOF problem in jacoco --- .github/workflows/core-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index d346fdee8a..6361cc6254 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -53,6 +53,10 @@ jobs: - name: Build all module with Maven run: mvn clean install -pl '!trino' -Dcheckstyle.skip=true + - name: debug jacoco EOF problems + if: ${{ always() }} + run: ls -lha /home/runner/work/arctic/arctic/flink/v1.12/flink/target/jacoco.exec + - name: Code coverage uses: codecov/codecov-action@v3 with: From 4c5d7f49dbd54ec3a2d63ad7de1dd77987196cb0 Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Wed, 12 Apr 2023 14:38:14 +0800 Subject: [PATCH 4/9] add -X for CI --- .github/workflows/core-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index 6361cc6254..eabafe8375 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -51,7 +51,7 @@ jobs: run: mvn validate - name: Build all module with Maven - run: mvn clean install -pl '!trino' -Dcheckstyle.skip=true + run: mvn clean install -pl '!trino' -Dcheckstyle.skip=true -X - name: debug jacoco EOF problems if: ${{ always() }} @@ -61,4 +61,4 @@ jobs: uses: codecov/codecov-action@v3 with: verbose: true - flags: core \ No newline at end of file + flags: core From f1c48b92ecf9616f0ff59b0ba920033e3fbf931f Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Wed, 12 Apr 2023 16:05:44 +0800 Subject: [PATCH 5/9] upgrade maven-surefire-plugin to 3.0.0 --- pom.xml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 7808ddb620..e0a8c3d09f 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ 3.2.4 4.3.0 4.3 - 3.0.0-M7 + 3.0.0 3.8.1 3.3.0 3.0.0 @@ -522,23 +522,23 @@ .scalafmt.conf - - - pom.xml - - - ${project.build.sourceEncoding} - true - true - 4 - - + + + pom.xml + + + ${project.build.sourceEncoding} + true + true + 4 + + - check + check From 376f3c7f78c6105a2e37b47cdc9cc54fe6db23c7 Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Wed, 12 Apr 2023 16:49:30 +0800 Subject: [PATCH 6/9] remove maven scalestyle check during test --- .github/workflows/core-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index eabafe8375..f995b546f1 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -48,7 +48,7 @@ jobs: cache: maven - name: Run code checkstyle - run: mvn validate + run: mvn validate -Dmaven.scalastyle.skip=true - name: Build all module with Maven run: mvn clean install -pl '!trino' -Dcheckstyle.skip=true -X From efd4345c925dfeebaf3c5e935a1a09930438ea5c Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Wed, 12 Apr 2023 16:59:16 +0800 Subject: [PATCH 7/9] remove validate --- .github/workflows/core-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index f995b546f1..c123d4d31a 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -47,11 +47,9 @@ jobs: distribution: 'temurin' cache: maven - - name: Run code checkstyle - run: mvn validate -Dmaven.scalastyle.skip=true - name: Build all module with Maven - run: mvn clean install -pl '!trino' -Dcheckstyle.skip=true -X + run: mvn clean install -pl '!trino' -X - name: debug jacoco EOF problems if: ${{ always() }} From e599148e7478be9ece85686a5f8ffbd94661df99 Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Wed, 12 Apr 2023 17:25:24 +0800 Subject: [PATCH 8/9] add shutdown hook --- .../netease/arctic/flink/FlinkTestBase.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/flink/v1.12/flink/src/test/java/com/netease/arctic/flink/FlinkTestBase.java b/flink/v1.12/flink/src/test/java/com/netease/arctic/flink/FlinkTestBase.java index 0439a57610..eaac96c367 100644 --- a/flink/v1.12/flink/src/test/java/com/netease/arctic/flink/FlinkTestBase.java +++ b/flink/v1.12/flink/src/test/java/com/netease/arctic/flink/FlinkTestBase.java @@ -317,13 +317,15 @@ protected static void commit(KeyedTable keyedTable, WriteResult result, boolean } } - protected static TaskWriter createKeyedTaskWriter(KeyedTable keyedTable, RowType rowType, - boolean base) { + protected static TaskWriter createKeyedTaskWriter( + KeyedTable keyedTable, RowType rowType, + boolean base) { return createKeyedTaskWriter(keyedTable, rowType, base, 3); } - protected static TaskWriter createKeyedTaskWriter(KeyedTable keyedTable, RowType rowType, - boolean base, long mask) { + protected static TaskWriter createKeyedTaskWriter( + KeyedTable keyedTable, RowType rowType, + boolean base, long mask) { ArcticRowDataTaskWriterFactory taskWriterFactory = new ArcticRowDataTaskWriterFactory(keyedTable, rowType, base); taskWriterFactory.setMask(mask); @@ -331,4 +333,18 @@ protected static TaskWriter createKeyedTaskWriter(KeyedTable keyedTable return taskWriterFactory.create(); } + static { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public synchronized void start() { + LOG.info("JACOCO SHUTDOWN HOOK START"); + super.start(); + } + + @Override + public void run() { + LOG.info("JACOCO SHUTDOWN HOOK RUN"); + } + }); + } } From b238fc11cd3f0c93239a001662d89148cd95480e Mon Sep 17 00:00:00 2001 From: BaiyangTX Date: Wed, 12 Apr 2023 17:57:48 +0800 Subject: [PATCH 9/9] add -B --- .github/workflows/core-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index c123d4d31a..16a2ed519e 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -49,7 +49,7 @@ jobs: - name: Build all module with Maven - run: mvn clean install -pl '!trino' -X + run: mvn clean install -pl '!trino' -B -X - name: debug jacoco EOF problems if: ${{ always() }}