Merge branch 'fix_hive_image_timeout' of github.com:yuqi1129/gravitin…

…o into fix_hive_image_timeout
yuqi1129 · May 29, 2024 · 3663982 · 3663982
2 parents 4e4c57e + 3660df8
commit 3663982
Show file tree

Hide file tree

Showing 113 changed files with 1,879 additions and 324 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -40,7 +40,6 @@ jobs:
               - meta/**
               - server/**
               - server-common/**
-              - spark-connector/**
               - trino-connector/**
               - web/**
               - docs/open-api/**
@@ -96,7 +95,7 @@ jobs:
       - name: Backend Integration Test
         id: integrationTest
         run: |
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPyClientITs -PskipFlinkITs
+          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPyClientITs -PskipFlinkITs -PskipSparkITs
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml
@@ -0,0 +1,109 @@
+name: Spark Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - .github/**
+              - api/**
+              - bin/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - clients/filesystem-hadoop3/**
+              - clients/filesystem-hadoop3-runtime/**
+              - common/**
+              - conf/**
+              - core/**
+              - dev/**
+              - gradle/**
+              - meta/**
+              - server/**
+              - server-common/**
+              - spark-connector/**
+              - docs/open-api/**
+              - build.gradle.kts
+              - gradle.properties
+              - gradlew
+              - setting.gradle.kts
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        architecture: [linux/amd64]
+        java-version: [ 8, 11, 17 ]
+        test-mode: [ embedded, deploy ]
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if : ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Setup debug Github Action
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
+        uses: csexton/debugger-action@master
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Spark Integration Test
+        id: integrationTest
+        run: |
+          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}
+          path: |
+            build/reports
+            spark-connector/v3.3/spark/build/spark-3.3-integration-test.log
+            spark-connector/v3.4/spark/build/spark-3.4-integration-test.log
+            spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
diff --git a/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java b/api/src/main/java/com/datastrato/gravitino/rel/types/Types.java
@@ -1070,6 +1070,11 @@ public boolean equals(Object o) {
     public int hashCode() {
       return Objects.hash(unparsedType);
     }
+
+    @Override
+    public String toString() {
+      return unparsedType;
+    }
   }
 
   /**

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -617,7 +617,7 @@ tasks {
   register("copySubprojectDependencies", Copy::class) {
     subprojects.forEach() {
       if (!it.name.startsWith("catalog") &&
-        !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark-connector") && it.name != "trino-connector" &&
+        !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && it.name != "trino-connector" &&
         it.name != "integration-test" && it.name != "bundled-catalog" && it.name != "flink-connector"
       ) {
         from(it.configurations.runtimeClasspath)
@@ -631,7 +631,7 @@ tasks {
       if (!it.name.startsWith("catalog") &&
         !it.name.startsWith("client") &&
         !it.name.startsWith("filesystem") &&
-        !it.name.startsWith("spark-connector") &&
+        !it.name.startsWith("spark") &&
         it.name != "trino-connector" &&
         it.name != "integration-test" &&
         it.name != "bundled-catalog" &&

diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts
@@ -11,7 +11,7 @@ plugins {
 }
 
 val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
-val sparkVersion: String = libs.versions.spark.get()
+val sparkVersion: String = libs.versions.spark34.get()
 val icebergVersion: String = libs.versions.iceberg.get()
 val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()
 

diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -11,7 +11,7 @@ plugins {
 }
 
 val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
-val sparkVersion: String = libs.versions.spark.get()
+val sparkVersion: String = libs.versions.spark34.get()
 val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
 val icebergVersion: String = libs.versions.iceberg.get()
 val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()

diff --git a/clients/client-python/requirements-dev.txt b/clients/client-python/requirements-dev.txt
@@ -1,8 +1,8 @@
 # Copyright 2024 Datastrato Pvt Ltd.
 # This software is licensed under the Apache License version 2.
-requests
-dataclasses-json
-pylint
-black
-twine
-coverage
+requests==2.32.2
+dataclasses-json==0.6.6
+pylint==3.2.2
+black==24.4.2
+twine==5.1.0
+coverage==7.5.1
diff --git a/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java b/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java
@@ -32,4 +32,6 @@ private ConfigConstants() {}
   public static final String VERSION_0_4_0 = "0.4.0";
   /** The version number for the 0.5.0 release. */
   public static final String VERSION_0_5_0 = "0.5.0";
+  /** The version number for the 0.5.1 release. */
+  public static final String VERSION_0_5_1 = "0.5.1";
 }
diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java
@@ -700,7 +700,7 @@ private static Type readDataType(JsonNode node) {
       }
     }
 
-    throw new IllegalArgumentException("Cannot parse type from JSON: " + node);
+    return Types.UnparsedType.of(node.toString());
   }
 
   private static void writeUnionType(Types.UnionType unionType, JsonGenerator gen)
@@ -780,7 +780,7 @@ private static void writeUnparsedType(String unparsedType, JsonGenerator gen) th
     gen.writeEndObject();
   }
 
-  private static Type.PrimitiveType fromPrimitiveTypeString(String typeString) {
+  private static Type fromPrimitiveTypeString(String typeString) {
     Type.PrimitiveType primitiveType = TYPES.get(typeString);
     if (primitiveType != null) {
       return primitiveType;
@@ -807,7 +807,7 @@ private static Type.PrimitiveType fromPrimitiveTypeString(String typeString) {
           Integer.parseInt(decimal.group(1)), Integer.parseInt(decimal.group(2)));
     }
 
-    throw new IllegalArgumentException("Cannot parse type string to primitiveType: " + typeString);
+    return Types.UnparsedType.of(typeString);
   }
 
   private static Types.StructType readStructType(JsonNode node) {

diff --git a/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java b/common/src/test/java/com/datastrato/gravitino/json/TestJsonUtils.java
@@ -162,6 +162,18 @@ public void testTypeSerDe() throws Exception {
     Assertions.assertEquals(objectMapper.readTree(expected), objectMapper.readTree(jsonValue));
   }
 
+  @Test
+  public void testTypeSerdeCompatibility() throws JsonProcessingException {
+    String newPrimitiveType = "\"new_primitive_type\"";
+    Type type = objectMapper.readValue(newPrimitiveType, Type.class);
+    Assertions.assertEquals(Types.UnparsedType.of("new_primitive_type"), type);
+
+    String newComplexType = "{\"type\": \"new_complex_type\", \"field\": \"value\"}";
+    type = objectMapper.readValue(newComplexType, Type.class);
+    Assertions.assertEquals(
+        Types.UnparsedType.of("{\"type\":\"new_complex_type\",\"field\":\"value\"}"), type);
+  }
+
   @Test
   void testGetLong() throws Exception {
     String jsonException = "{\"property\": \"value\"}";

diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java
@@ -276,4 +276,12 @@ private Configs() {}
           .version(ConfigConstants.VERSION_0_5_0)
           .longConf()
           .createWithDefault(60 * 60 * 1000L);
+
+  public static final int DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS = 60;
+  public static final ConfigEntry<Integer> METRICS_TIME_SLIDING_WINDOW_SECONDS =
+      new ConfigBuilder("gravitino.metrics.timeSlidingWindowSecs")
+          .doc("The seconds of Gravitino metrics time sliding window")
+          .version(ConfigConstants.VERSION_0_5_1)
+          .intConf()
+          .createWithDefault(DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS);
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java b/core/src/main/java/com/datastrato/gravitino/metrics/source/MetricsSource.java
@@ -9,7 +9,12 @@
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SlidingTimeWindowArrayReservoir;
 import com.codahale.metrics.Timer;
+import com.datastrato.gravitino.Config;
+import com.datastrato.gravitino.Configs;
+import com.datastrato.gravitino.GravitinoEnv;
+import java.util.concurrent.TimeUnit;
 
 /**
  * MetricsSource provides utilities to collect specified kind metrics, all metrics must create with
@@ -24,10 +29,19 @@ public abstract class MetricsSource {
   public static final String JVM_METRIC_NAME = "jvm";
   private final MetricRegistry metricRegistry;
   private final String metricsSourceName;
+  private final int timeSlidingWindowSeconds;
 
   protected MetricsSource(String name) {
     this.metricsSourceName = name;
     metricRegistry = new MetricRegistry();
+    Config config = GravitinoEnv.getInstance().config();
+    if (config != null) {
+      this.timeSlidingWindowSeconds =
+          config.get(Configs.METRICS_TIME_SLIDING_WINDOW_SECONDS).intValue();
+    } else {
+      // Couldn't get config when testing
+      this.timeSlidingWindowSeconds = Configs.DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS;
+    }
   }
 
   /**
@@ -75,7 +89,12 @@ public Counter getCounter(String name) {
    * @return a new or pre-existing Histogram
    */
   public Histogram getHistogram(String name) {
-    return this.metricRegistry.histogram(name);
+    return this.metricRegistry.histogram(
+        name,
+        () ->
+            new Histogram(
+                new SlidingTimeWindowArrayReservoir(
+                    getTimeSlidingWindowSeconds(), TimeUnit.SECONDS)));
   }
 
   /**
@@ -85,6 +104,15 @@ public Histogram getHistogram(String name) {
    * @return a new or pre-existing Timer
    */
   public Timer getTimer(String name) {
-    return this.metricRegistry.timer(name);
+    return this.metricRegistry.timer(
+        name,
+        () ->
+            new Timer(
+                new SlidingTimeWindowArrayReservoir(
+                    getTimeSlidingWindowSeconds(), TimeUnit.SECONDS)));
+  }
+
+  protected int getTimeSlidingWindowSeconds() {
+    return timeSlidingWindowSeconds;
   }
 }
diff --git a/core/src/test/java/com/datastrato/gravitino/metrics/TestMetricsSystem.java b/core/src/test/java/com/datastrato/gravitino/metrics/TestMetricsSystem.java
@@ -10,11 +10,7 @@
 import org.junit.jupiter.api.Test;
 
 public class TestMetricsSystem {
-  MetricsSystem metricsSystem = new MetricsSystem();
-
-  private long getCounterValue(String metricsSourceName, String name) {
-    return metricsSystem.getMetricRegistry().counter(metricsSourceName + "." + name).getCount();
-  }
+  private MetricsSystem metricsSystem = new MetricsSystem();
 
   @Test
   void testRegisterMetricsSource() {
@@ -53,4 +49,8 @@ void testRegisterMetricsSource() {
             .getCounters()
             .containsKey(metricsSource.getMetricsSourceName() + "a.b"));
   }
+
+  private long getCounterValue(String metricsSourceName, String name) {
+    return metricsSystem.getMetricRegistry().counter(metricsSourceName + "." + name).getCount();
+  }
 }
diff --git a/core/src/test/java/com/datastrato/gravitino/metrics/source/TestMetricsSource.java b/core/src/test/java/com/datastrato/gravitino/metrics/source/TestMetricsSource.java
@@ -28,7 +28,7 @@ public TestMetricsSource() {
 
   @BeforeAll
   void init() {
-    metricsSystem = new MetricsSystem();
+    this.metricsSystem = new MetricsSystem();
     metricsSystem.register(this);
   }
 

diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md
@@ -131,6 +131,12 @@ For more details, please refer to the definition of the plugin.
 
 Refer to [security](security.md) for HTTPS and authentication configurations.
 
+### Metrics configuration
+
+| Property name                             | Description                                          | Default value | Required | Since Version |
+|-------------------------------------------|------------------------------------------------------|---------------|----------|---------------|
+| `gravitino.metrics.timeSlidingWindowSecs` | The seconds of Gravitino metrics time sliding window | 60            | No       | 0.5.1         |
+
 ## Gravitino catalog properties configuration
 
 There are three types of catalog properties:

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -27,9 +27,15 @@ commons-collections4 = "4.4"
 commons-dbcp2 = "2.11.0"
 caffeine = "2.9.3"
 rocksdbjni = "7.10.2"
-iceberg = '1.3.1' # 1.4.0 causes test to fail
+iceberg = '1.3.1' # used for Gravitino Iceberg catalog and Iceberg REST service
+iceberg4spark = "1.4.1" # used for compile spark connector
 trino = '426'
-spark = "3.4.1" # 3.5.0 causes tests to fail
+spark33 = "3.3.4" 
+spark34 = "3.4.3"
+spark35 = "3.5.1"
+kyuubi4spark33 = "1.7.4"
+kyuubi4spark34 = "1.8.2"
+kyuubi4spark35 = "1.9.0"
 scala-collection-compat = "2.7.0"
 scala-java-compat = "1.0.2"
 sqlite-jdbc = "3.42.0.0"
@@ -49,7 +55,6 @@ selenium = "3.141.59"
 rauschig = "1.2.0"
 mybatis = "3.5.6"
 h2db = "1.4.200"
-kyuubi = "1.8.2"
 kafka = "3.4.0"
 curator = "2.12.0"
 awaitility = "4.2.1"