feat: add built-in udfs in config (databendlabs#15938)

* chore: spill out built-in users and udfs

* fix the unit test

* add unit test

* add udf parser to UserDefinedFunction

* add sqllogic test

* fix sqllogic test

* idm -> builtin

* ignore the invalid built-in user&udf

* add built-in to toml config

scripts/ci/deploy/config/databend-query-node-1.toml

@@ -66,6 +66,11 @@ auth_type = "no_password"
 # #  echo -n "datafuselabs" | sha256sum
 # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [query.settings]
 aggregate_spilling_memory_ratio = 60
 join_spilling_memory_ratio = 60

scripts/ci/deploy/config/databend-query-node-2.toml

@@ -46,6 +46,11 @@ auth_type = "no_password"
 name = "default"
 auth_type = "no_password"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [log]
 
 [log.file]

scripts/ci/deploy/config/databend-query-node-3.toml

@@ -47,6 +47,11 @@ auth_type = "no_password"
 name = "default"
 auth_type = "no_password"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [log]
 
 [log.file]

scripts/ci/deploy/config/databend-query-node-hive.toml

@@ -42,6 +42,11 @@ auth_type = "no_password"
 name = "default"
 auth_type = "no_password"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [log]
 
 [log.file]

scripts/ci/deploy/config/databend-query-node-native.toml

@@ -59,6 +59,11 @@ auth_type = "no_password"
 # #  echo -n "datafuselabs" | sha256sum
 # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 
 [log]
 

scripts/ci/deploy/config/databend-query-node-otlp-logs.toml

@@ -38,6 +38,11 @@ auth_type = "no_password"
 name = "default"
 auth_type = "no_password"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [log]
 
 [log.file]

scripts/ci/deploy/config/databend-query-node-share-1.toml

@@ -61,6 +61,10 @@ auth_type = "no_password"
 # #  echo -n "datafuselabs" | sha256sum
 # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
 
 [log]
 

scripts/ci/deploy/config/databend-query-node-share-2.toml

@@ -63,6 +63,11 @@ auth_type = "no_password"
 # #  echo -n "datafuselabs" | sha256sum
 # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [log]
 
 [log.file]

scripts/ci/deploy/config/databend-query-node-share-3.toml

@@ -63,6 +63,11 @@ auth_type = "no_password"
 # #  echo -n "datafuselabs" | sha256sum
 # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
 
+# This for test
+[[query.udfs]]
+name = "ping"
+definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'"
+
 [log]
 
 [log.file]

src/binaries/query/entry.rs

@@ -286,10 +286,21 @@ pub async fn start_services(conf: &InnerConfig) -> Result<()> {
     println!(
         "Builtin users: {}",
         conf.query
-            .idm
+            .builtin
             .users
-            .keys()
-            .map(|name| name.to_string())
+            .iter()
+            .map(|config| config.name.clone())
+            .collect::<Vec<_>>()
+            .join(", ")
+    );
+    println!();
+    println!(
+        "Builtin UDFs: {}",
+        conf.query
+            .builtin
+            .udfs
+            .iter()
+            .map(|config| config.name.clone())
             .collect::<Vec<_>>()
             .join(", ")
     );

src/query/config/Cargo.toml

@@ -27,8 +27,6 @@ databend-common-grpc = { workspace = true }
 databend-common-meta-app = { workspace = true }
 databend-common-storage = { workspace = true }
 databend-common-tracing = { workspace = true }
-databend-common-users = { workspace = true }
-hex = "0.4.3"
 log = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }

src/query/config/src/builtin.rs

@@ -0,0 +1,41 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct BuiltInConfig {
+    pub users: Vec<UserConfig>,
+    pub udfs: Vec<UDFConfig>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct UserConfig {
+    pub name: String,
+    #[serde(flatten)]
+    pub auth: UserAuthConfig,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct UserAuthConfig {
+    pub auth_type: String,
+    pub auth_string: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct UDFConfig {
+    pub name: String,
+    pub definition: String,
+}