diff --git a/Cargo.lock b/Cargo.lock index f2c5bf6fa2836..1a0c081c4c0b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3331,8 +3331,6 @@ dependencies = [ "databend-common-meta-app", "databend-common-storage", "databend-common-tracing", - "databend-common-users", - "hex", "log", "pretty_assertions", "semver", diff --git a/scripts/ci/deploy/config/databend-query-node-1.toml b/scripts/ci/deploy/config/databend-query-node-1.toml index 1ba43c61df10d..5d2f666934263 100644 --- a/scripts/ci/deploy/config/databend-query-node-1.toml +++ b/scripts/ci/deploy/config/databend-query-node-1.toml @@ -66,6 +66,11 @@ auth_type = "no_password" # # echo -n "datafuselabs" | sha256sum # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [query.settings] aggregate_spilling_memory_ratio = 60 join_spilling_memory_ratio = 60 diff --git a/scripts/ci/deploy/config/databend-query-node-2.toml b/scripts/ci/deploy/config/databend-query-node-2.toml index 3398d8598e569..9a4f02ab4e35f 100644 --- a/scripts/ci/deploy/config/databend-query-node-2.toml +++ b/scripts/ci/deploy/config/databend-query-node-2.toml @@ -46,6 +46,11 @@ auth_type = "no_password" name = "default" auth_type = "no_password" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] [log.file] diff --git a/scripts/ci/deploy/config/databend-query-node-3.toml b/scripts/ci/deploy/config/databend-query-node-3.toml index e3fffa63bbeec..c97e498a5fe69 100644 --- a/scripts/ci/deploy/config/databend-query-node-3.toml +++ b/scripts/ci/deploy/config/databend-query-node-3.toml @@ -47,6 +47,11 @@ auth_type = "no_password" name = "default" auth_type = "no_password" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] [log.file] diff --git a/scripts/ci/deploy/config/databend-query-node-hive.toml b/scripts/ci/deploy/config/databend-query-node-hive.toml index 13043be9cefb6..b4a115f59e60a 100644 --- a/scripts/ci/deploy/config/databend-query-node-hive.toml +++ b/scripts/ci/deploy/config/databend-query-node-hive.toml @@ -42,6 +42,11 @@ auth_type = "no_password" name = "default" auth_type = "no_password" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] [log.file] diff --git a/scripts/ci/deploy/config/databend-query-node-native.toml b/scripts/ci/deploy/config/databend-query-node-native.toml index 5c16bedeb0f38..2ded7654521d1 100644 --- a/scripts/ci/deploy/config/databend-query-node-native.toml +++ b/scripts/ci/deploy/config/databend-query-node-native.toml @@ -59,6 +59,11 @@ auth_type = "no_password" # # echo -n "datafuselabs" | sha256sum # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] diff --git a/scripts/ci/deploy/config/databend-query-node-otlp-logs.toml b/scripts/ci/deploy/config/databend-query-node-otlp-logs.toml index baafc3a0476ba..ad9abf5cdd292 100644 --- a/scripts/ci/deploy/config/databend-query-node-otlp-logs.toml +++ b/scripts/ci/deploy/config/databend-query-node-otlp-logs.toml @@ -38,6 +38,11 @@ auth_type = "no_password" name = "default" auth_type = "no_password" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] [log.file] diff --git a/scripts/ci/deploy/config/databend-query-node-share-1.toml b/scripts/ci/deploy/config/databend-query-node-share-1.toml index 9c22a5338b0ad..ef92a52f8048a 100644 --- a/scripts/ci/deploy/config/databend-query-node-share-1.toml +++ b/scripts/ci/deploy/config/databend-query-node-share-1.toml @@ -61,6 +61,10 @@ auth_type = "no_password" # # echo -n "datafuselabs" | sha256sum # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" [log] diff --git a/scripts/ci/deploy/config/databend-query-node-share-2.toml b/scripts/ci/deploy/config/databend-query-node-share-2.toml index 9fb83e798b2b6..7c7991ea21f5f 100644 --- a/scripts/ci/deploy/config/databend-query-node-share-2.toml +++ b/scripts/ci/deploy/config/databend-query-node-share-2.toml @@ -63,6 +63,11 @@ auth_type = "no_password" # # echo -n "datafuselabs" | sha256sum # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] [log.file] diff --git a/scripts/ci/deploy/config/databend-query-node-share-3.toml b/scripts/ci/deploy/config/databend-query-node-share-3.toml index 2318bbc47d811..9c049abcdf7ce 100644 --- a/scripts/ci/deploy/config/databend-query-node-share-3.toml +++ b/scripts/ci/deploy/config/databend-query-node-share-3.toml @@ -63,6 +63,11 @@ auth_type = "no_password" # # echo -n "datafuselabs" | sha256sum # auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b" +# This for test +[[query.udfs]] +name = "ping" +definition = "CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'" + [log] [log.file] diff --git a/src/binaries/query/entry.rs b/src/binaries/query/entry.rs index c82e7c157680c..c0a76ac4463a5 100644 --- a/src/binaries/query/entry.rs +++ b/src/binaries/query/entry.rs @@ -286,10 +286,21 @@ pub async fn start_services(conf: &InnerConfig) -> Result<()> { println!( "Builtin users: {}", conf.query - .idm + .builtin .users - .keys() - .map(|name| name.to_string()) + .iter() + .map(|config| config.name.clone()) + .collect::>() + .join(", ") + ); + println!(); + println!( + "Builtin UDFs: {}", + conf.query + .builtin + .udfs + .iter() + .map(|config| config.name.clone()) .collect::>() .join(", ") ); diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index 8a1cc64f910b1..f6ad61dc11bbf 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -27,8 +27,6 @@ databend-common-grpc = { workspace = true } databend-common-meta-app = { workspace = true } databend-common-storage = { workspace = true } databend-common-tracing = { workspace = true } -databend-common-users = { workspace = true } -hex = "0.4.3" log = { workspace = true } semver = { workspace = true } serde = { workspace = true } diff --git a/src/query/config/src/builtin.rs b/src/query/config/src/builtin.rs new file mode 100644 index 0000000000000..9c9ce6481d3e2 --- /dev/null +++ b/src/query/config/src/builtin.rs @@ -0,0 +1,41 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::Deserialize; +use serde::Serialize; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BuiltInConfig { + pub users: Vec, + pub udfs: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct UserConfig { + pub name: String, + #[serde(flatten)] + pub auth: UserAuthConfig, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct UserAuthConfig { + pub auth_type: String, + pub auth_string: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct UDFConfig { + pub name: String, + pub definition: String, +} diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 35e5cdb836b72..5f08cd5473ed7 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -18,7 +18,6 @@ use std::env; use std::fmt; use std::fmt::Debug; use std::fmt::Formatter; -use std::str::FromStr; use clap::ArgAction; use clap::Args; @@ -28,8 +27,6 @@ use clap::ValueEnum; use databend_common_base::base::mask_string; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::principal::AuthInfo; -use databend_common_meta_app::principal::AuthType; use databend_common_meta_app::principal::UserSettingValue; use databend_common_meta_app::storage::StorageAzblobConfig as InnerStorageAzblobConfig; use databend_common_meta_app::storage::StorageCosConfig as InnerStorageCosConfig; @@ -55,7 +52,6 @@ use databend_common_tracing::QueryLogConfig as InnerQueryLogConfig; use databend_common_tracing::StderrConfig as InnerStderrLogConfig; use databend_common_tracing::StructLogConfig as InnerStructLogConfig; use databend_common_tracing::TracingConfig as InnerTracingConfig; -use databend_common_users::idm_config::IDMConfig as InnerIDMConfig; use serde::Deserialize; use serde::Serialize; use serde_with::with_prefix; @@ -72,6 +68,9 @@ use super::inner::LocalConfig as InnerLocalConfig; use super::inner::MetaConfig as InnerMetaConfig; use super::inner::QueryConfig as InnerQueryConfig; use crate::background_config::BackgroundConfig; +use crate::builtin::BuiltInConfig; +use crate::builtin::UDFConfig; +use crate::builtin::UserConfig; use crate::DATABEND_COMMIT_VERSION; const CATALOG_HIVE: &str = "hive"; @@ -88,7 +87,7 @@ const CATALOG_HIVE: &str = "hive"; /// Only adding new fields is allowed. /// This same rules should be applied to all fields of this struct. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Parser)] -#[clap(name = "databend-query", about, version = &**DATABEND_COMMIT_VERSION, author)] +#[clap(name = "databend-query", about, version = & * * DATABEND_COMMIT_VERSION, author)] #[serde(default)] pub struct Config { /// Run a command and quit @@ -1407,7 +1406,12 @@ pub struct QueryConfig { #[clap(long, value_name = "VALUE", default_value = "0")] pub max_server_memory_usage: u64, - #[clap(long, value_name = "VALUE",value_parser = clap::value_parser!(bool), default_value = "false")] + #[clap( + long, + value_name = "VALUE", + value_parser = clap::value_parser!(bool), + default_value = "false" + )] pub max_memory_limit_enabled: bool, #[deprecated(note = "clickhouse tcp support is deprecated")] @@ -1491,7 +1495,12 @@ pub struct QueryConfig { pub rpc_client_timeout_secs: u64, /// Table engine memory enabled - #[clap(long, value_name = "VALUE",value_parser = clap::value_parser!(bool), default_value = "true")] + #[clap( + long, + value_name = "VALUE", + value_parser = clap::value_parser!(bool), + default_value = "true" + )] pub table_engine_memory_enabled: bool, #[clap(long, value_name = "VALUE", default_value = "5000")] @@ -1523,6 +1532,9 @@ pub struct QueryConfig { #[clap(skip)] users: Vec, + #[clap(skip)] + udfs: Vec, + #[clap(long, value_name = "VALUE", default_value = "")] pub share_endpoint_address: String, @@ -1721,8 +1733,9 @@ impl TryInto for QueryConfig { jwt_key_files: self.jwt_key_files, default_storage_format: self.default_storage_format, default_compression: self.default_compression, - idm: InnerIDMConfig { - users: users_to_inner(self.users)?, + builtin: BuiltInConfig { + users: self.users, + udfs: self.udfs, }, share_endpoint_address: self.share_endpoint_address, share_endpoint_auth_token_file: self.share_endpoint_auth_token_file, @@ -1806,7 +1819,8 @@ impl From for QueryConfig { jwt_key_files: inner.jwt_key_files, default_storage_format: inner.default_storage_format, default_compression: inner.default_compression, - users: users_from_inner(inner.idm.users), + users: inner.builtin.users, + udfs: inner.builtin.udfs, share_endpoint_address: inner.share_endpoint_address, share_endpoint_auth_token_file: inner.share_endpoint_auth_token_file, quota: inner.tenant_quota, @@ -2014,7 +2028,9 @@ impl From for LogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct FileLogConfig { - #[clap(long = "log-file-on", value_name = "VALUE", default_value = "true", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-file-on", value_name = "VALUE", default_value = "true", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub file_on: bool, @@ -2091,7 +2107,9 @@ impl From for FileLogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct StderrLogConfig { - #[clap(long = "log-stderr-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-stderr-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub stderr_on: bool, @@ -2144,7 +2162,9 @@ impl From for StderrLogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct OTLPLogConfig { - #[clap(long = "log-otlp-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-otlp-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub otlp_on: bool, @@ -2189,7 +2209,9 @@ impl From for OTLPLogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct QueryLogConfig { - #[clap(long = "log-query-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-query-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub log_query_on: bool, @@ -2234,7 +2256,9 @@ impl From for QueryLogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct ProfileLogConfig { - #[clap(long = "log-profile-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-profile-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub log_profile_on: bool, @@ -2282,7 +2306,9 @@ impl From for ProfileLogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct StructLogConfig { - #[clap(long = "log-structlog-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-structlog-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub log_structlog_on: bool, @@ -2321,7 +2347,9 @@ impl From for StructLogConfig { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct TracingConfig { - #[clap(long = "log-tracing-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true")] + #[clap( + long = "log-tracing-on", value_name = "VALUE", default_value = "false", action = ArgAction::Set, num_args = 0..=1, require_equals = true, default_missing_value = "true" + )] #[serde(rename = "on")] pub tracing_on: bool, @@ -2583,90 +2611,6 @@ impl Debug for MetaConfig { } } -fn users_from_inner(inner: HashMap) -> Vec { - inner - .into_iter() - .map(|(name, auth)| UserConfig { - name, - auth: auth.into(), - }) - .collect() -} - -fn users_to_inner(outer: Vec) -> Result> { - let mut inner = HashMap::new(); - for c in outer.into_iter() { - inner.insert(c.name.clone(), c.auth.try_into()?); - } - Ok(inner) -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct UserConfig { - pub name: String, - #[serde(flatten)] - pub auth: UserAuthConfig, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct UserAuthConfig { - auth_type: String, - auth_string: Option, -} - -fn check_no_auth_string(auth_string: Option, auth_info: AuthInfo) -> Result { - match auth_string { - Some(s) if !s.is_empty() => Err(ErrorCode::InvalidConfig(format!( - "should not set auth_string for auth_type {}", - auth_info.get_type().to_str() - ))), - _ => Ok(auth_info), - } -} - -impl TryInto for UserAuthConfig { - type Error = ErrorCode; - - fn try_into(self) -> Result { - let auth_type = AuthType::from_str(&self.auth_type)?; - match auth_type { - AuthType::NoPassword => check_no_auth_string(self.auth_string, AuthInfo::None), - AuthType::JWT => check_no_auth_string(self.auth_string, AuthInfo::JWT), - AuthType::Sha256Password | AuthType::DoubleSha1Password => { - let password_type = auth_type.get_password_type().expect("must success"); - match self.auth_string { - None => Err(ErrorCode::InvalidConfig("must set auth_string")), - Some(s) => { - let p = hex::decode(s).map_err(|e| { - ErrorCode::InvalidConfig(format!("password is not hex: {e:?}")) - })?; - Ok(AuthInfo::Password { - hash_value: p, - hash_method: password_type, - }) - } - } - } - } - } -} - -impl From for UserAuthConfig { - fn from(inner: AuthInfo) -> Self { - let auth_type = inner.get_type().to_str().to_owned(); - let auth_string = inner.get_auth_string(); - let auth_string = if auth_string.is_empty() { - None - } else { - Some(hex::encode(auth_string)) - }; - UserAuthConfig { - auth_type, - auth_string, - } - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct LocalConfig { diff --git a/src/query/config/src/inner.rs b/src/query/config/src/inner.rs index f46001fc380b5..ef99ca3cc6055 100644 --- a/src/query/config/src/inner.rs +++ b/src/query/config/src/inner.rs @@ -31,11 +31,11 @@ use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant::TenantQuota; use databend_common_storage::StorageConfig; use databend_common_tracing::Config as LogConfig; -use databend_common_users::idm_config::IDMConfig; use super::config::Commands; use super::config::Config; use crate::background_config::InnerBackgroundConfig; +use crate::BuiltInConfig; /// Inner config for query. /// @@ -210,7 +210,7 @@ pub struct QueryConfig { pub jwt_key_files: Vec, pub default_storage_format: String, pub default_compression: String, - pub idm: IDMConfig, + pub builtin: BuiltInConfig, pub share_endpoint_address: String, pub share_endpoint_auth_token_file: String, pub tenant_quota: Option, @@ -288,7 +288,7 @@ impl Default for QueryConfig { jwt_key_files: Vec::new(), default_storage_format: "auto".to_string(), default_compression: "auto".to_string(), - idm: IDMConfig::default(), + builtin: BuiltInConfig::default(), share_endpoint_address: "".to_string(), share_endpoint_auth_token_file: "".to_string(), tenant_quota: None, @@ -614,6 +614,7 @@ pub enum DiskCacheKeyReloadPolicy { // but cache capacity will not be checked Fuzzy, } + impl Default for DiskCacheKeyReloadPolicy { fn default() -> Self { Self::Reset diff --git a/src/query/config/src/lib.rs b/src/query/config/src/lib.rs index 9fbb452ce70dd..f1905503f45f1 100644 --- a/src/query/config/src/lib.rs +++ b/src/query/config/src/lib.rs @@ -17,6 +17,7 @@ #![feature(lazy_cell)] mod background_config; +mod builtin; /// Config mods provide config support. /// /// We are providing two config types: @@ -36,6 +37,7 @@ mod mask; mod obsolete; mod version; +pub use builtin::*; pub use config::CacheStorageTypeConfig; pub use config::Commands; pub use config::Config; diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index d958af0c419a9..7e7db97c9dbee 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -125,6 +125,7 @@ flatbuffers = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } headers = "0.4.0" +hex = "0.4.3" highway = "1.1" http = { workspace = true } humantime = "2.1.0" diff --git a/src/query/service/src/builtin/builtin_udfs.rs b/src/query/service/src/builtin/builtin_udfs.rs new file mode 100644 index 0000000000000..8a091e80e7ee9 --- /dev/null +++ b/src/query/service/src/builtin/builtin_udfs.rs @@ -0,0 +1,156 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use databend_common_ast::ast::Statement; +use databend_common_ast::ast::UDFDefinition; +use databend_common_ast::parser::parse_sql; +use databend_common_ast::parser::tokenize_sql; +use databend_common_ast::parser::Dialect; +use databend_common_config::UDFConfig; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::types::DataType; +use databend_common_meta_app::principal::UserDefinedFunction; +use databend_common_sql::resolve_type_name; +use log::error; + +pub struct BuiltinUDFs { + udf_configs: Vec, +} + +impl BuiltinUDFs { + pub fn create(udf_configs: Vec) -> BuiltinUDFs { + BuiltinUDFs { udf_configs } + } + + // Parse the UDF definition and return the UserDefinedFunction. + fn parse_udf_definition(&self, name: &str, definition: &str) -> Result { + let tokens = tokenize_sql(definition)?; + let (stmt, _) = parse_sql(&tokens, Dialect::PostgreSQL)?; + + match stmt { + Statement::CreateUDF(ast) => match ast.definition { + UDFDefinition::UDFServer { + arg_types, + return_type, + address, + handler, + language, + } => { + let mut arg_datatypes = Vec::with_capacity(arg_types.len()); + for arg_type in arg_types { + arg_datatypes.push(DataType::from(&resolve_type_name(&arg_type, true)?)); + } + let return_type = DataType::from(&resolve_type_name(&return_type, true)?); + let udf = UserDefinedFunction::create_udf_server( + name, + &address, + &handler, + &language, + arg_datatypes, + return_type, + "Built-in UDF", + ); + + Ok(udf) + } + _ => Err(ErrorCode::SyntaxException(format!( + "Invalid built-in UDF definition: '{}', expected UDFServer but got: {:?}", + definition, ast + ))), + }, + _ => Err(ErrorCode::SyntaxException(format!( + "Invalid built-in UDF definition: '{}', expected CreateUDF but got: {:?}", + definition, stmt + ))), + } + } + + /// Convert to UDFs. + /// Skip invalid UDF configs. + pub fn to_udfs(&self) -> HashMap { + let mut udf_map = HashMap::new(); + + for udf_config in self.udf_configs.iter() { + match self.parse_udf_definition(&udf_config.name, &udf_config.definition) { + Ok(user_defined_function) => { + udf_map.insert(user_defined_function.name.clone(), user_defined_function); + } + Err(e) => { + error!( + "Failed to parse built-in UDF definition for '{}': {}", + udf_config.name, e + ); + } + } + } + + udf_map + } +} + +#[cfg(test)] +mod tests { + use tokio::test; + + use super::*; + + #[test] + async fn test_to_meta_udfs() { + // Array of mock data for different test cases + let udf_configs = vec![ + UDFConfig { + name: "test_udf1".to_string(), + definition: "CREATE OR REPLACE FUNCTION test_udf1(STRING) + RETURNS STRING + LANGUAGE python +HANDLER = 'test_udf1' +ADDRESS = 'https://databend.com'" + .to_string(), + }, + UDFConfig { + name: "test_udf2".to_string(), + definition: "CREATE OR REPLACE FUNCTION test_udf2(ARRAY(FLOAT), ARRAY(FLOAT)) + RETURNS FLOAT + LANGUAGE python +HANDLER = 'test_udf2' +ADDRESS = 'https://databend.com'" + .to_string(), + }, + UDFConfig { + name: "invalid_udf".to_string(), + definition: "SELECT 1".to_string(), + }, + ]; + + let builtin_udfs = BuiltinUDFs::create(udf_configs); + + let udf_map = builtin_udfs.to_udfs(); + + // Test first UDF + assert!(udf_map.contains_key("test_udf1")); + let udf1 = udf_map.get("test_udf1").unwrap(); + assert_eq!(udf1.name, "test_udf1"); + + // Test second UDF + assert!(udf_map.contains_key("test_udf2")); + let udf2 = udf_map.get("test_udf2").unwrap(); + assert_eq!(udf2.name, "test_udf2"); + + // Test invalid UDF is not included + assert!(!udf_map.contains_key("invalid_udf")); + } +} diff --git a/src/query/service/src/builtin/builtin_users.rs b/src/query/service/src/builtin/builtin_users.rs new file mode 100644 index 0000000000000..75fe7957ac1b1 --- /dev/null +++ b/src/query/service/src/builtin/builtin_users.rs @@ -0,0 +1,217 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::str::FromStr; + +use databend_common_config::UserConfig; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_meta_app::principal::AuthInfo; +use databend_common_meta_app::principal::AuthType; +use log::error; + +pub struct BuiltinUsers { + user_configs: Vec, +} + +impl BuiltinUsers { + pub fn create(user_configs: Vec) -> BuiltinUsers { + BuiltinUsers { user_configs } + } + + fn check_no_auth_string(auth_string: Option, auth_info: AuthInfo) -> Result { + match auth_string { + Some(s) if !s.is_empty() => Err(ErrorCode::InvalidConfig(format!( + "should not set auth_string for auth_type {}", + auth_info.get_type().to_str() + ))), + _ => Ok(auth_info), + } + } + + fn parse_user_auth_config(user_config: UserConfig) -> Result { + let auth_config = user_config.auth.clone(); + let auth_type = AuthType::from_str(&auth_config.auth_type)?; + match auth_type { + AuthType::NoPassword => { + Self::check_no_auth_string(auth_config.auth_string.clone(), AuthInfo::None) + } + AuthType::JWT => { + Self::check_no_auth_string(auth_config.auth_string.clone(), AuthInfo::JWT) + } + AuthType::Sha256Password | AuthType::DoubleSha1Password => { + let password_type = auth_type.get_password_type().expect("must success"); + match &auth_config.auth_string { + None => Err(ErrorCode::InvalidConfig("must set auth_string")), + Some(s) => { + let p = hex::decode(s).map_err(|e| { + ErrorCode::InvalidConfig(format!("password is not hex: {e:?}")) + })?; + Ok(AuthInfo::Password { + hash_value: p, + hash_method: password_type, + }) + } + } + } + } + } + + /// Convert to auth infos. + /// Skip invalid user auth configs. + pub fn to_auth_infos(&self) -> HashMap { + let mut auth_infos = HashMap::new(); + for user_config in self.user_configs.iter() { + match Self::parse_user_auth_config(user_config.clone()) { + Ok(auth_info) => { + auth_infos.insert(user_config.name.clone(), auth_info); + } + Err(e) => { + error!( + "Failed to parse built-in user auth for '{}': {}", + user_config.name, e + ); + } + } + } + auth_infos + } +} + +#[cfg(test)] +mod tests { + use databend_common_config::UserAuthConfig; + use databend_common_meta_app::principal::PasswordHashMethod; + + use super::*; + + // Helper function to create a UserConfig for testing + fn create_user_config(name: &str, auth_type: &str, auth_string: Option) -> UserConfig { + UserConfig { + name: name.to_string(), + auth: UserAuthConfig { + auth_type: auth_type.to_string(), + auth_string, + }, + } + } + + #[test] + fn test_no_password_user() { + let user_configs = vec![create_user_config("user1", "no_password", None)]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + let auth_info = auth_infos.get("user1").unwrap(); + + assert_eq!(auth_info.get_type(), AuthType::NoPassword); + } + + #[test] + fn test_jwt_user() { + let user_configs = vec![create_user_config("user2", "jwt", None)]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + let auth_info = auth_infos.get("user2").unwrap(); + + assert_eq!(auth_info.get_type(), AuthType::JWT); + } + + #[test] + fn test_sha256_password_user() { + let user_configs = vec![create_user_config( + "user3", + "sha256_password", + Some("5e884898da28047151d0e56f8dc6292773603d0d6aabbddde4208fdfb800bde3".to_string()), + )]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + let auth_info = auth_infos.get("user3").unwrap(); + + match auth_info { + AuthInfo::Password { + hash_value, + hash_method, + } => { + assert_eq!(hash_method, &PasswordHashMethod::Sha256); + assert_eq!( + hex::encode(hash_value), + "5e884898da28047151d0e56f8dc6292773603d0d6aabbddde4208fdfb800bde3" + ); + } + _ => panic!("Unexpected auth info type"), + } + } + + #[test] + fn test_double_sha1_password_user() { + let user_configs = vec![create_user_config( + "user4", + "double_sha1_password", + Some("8bff94b1c58e7733cb1bc36d385bb4986bffeb17".to_string()), + )]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + let auth_info = auth_infos.get("user4").unwrap(); + + match auth_info { + AuthInfo::Password { + hash_value, + hash_method, + } => { + assert_eq!(hash_method, &PasswordHashMethod::DoubleSha1); + assert_eq!( + hex::encode(hash_value), + "8bff94b1c58e7733cb1bc36d385bb4986bffeb17" + ); + } + _ => panic!("Unexpected auth info type"), + } + } + + #[test] + fn test_invalid_auth_string() { + let user_configs = vec![create_user_config( + "user5", + "sha256_password", + Some("invalid_hex".to_string()), + )]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + assert!(auth_infos.get("user5").is_none()); + } + + #[test] + fn test_missing_auth_string_for_password() { + let user_configs = vec![create_user_config("user6", "sha256_password", None)]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + assert!(auth_infos.get("user6").is_none()); + } + + #[test] + fn test_invalid_auth_type() { + let user_configs = vec![create_user_config("user7", "InvalidAuthType", None)]; + let builtin_users = BuiltinUsers::create(user_configs); + + let auth_infos = builtin_users.to_auth_infos(); + assert!(auth_infos.get("user7").is_none()); + } +} diff --git a/src/query/service/src/builtin/mod.rs b/src/query/service/src/builtin/mod.rs new file mode 100644 index 0000000000000..7551c0f66cd22 --- /dev/null +++ b/src/query/service/src/builtin/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod builtin_udfs; +mod builtin_users; + +pub use builtin_udfs::BuiltinUDFs; +pub use builtin_users::BuiltinUsers; diff --git a/src/query/service/src/global_services.rs b/src/query/service/src/global_services.rs index 93df6d01714bb..7604a4b51b24d 100644 --- a/src/query/service/src/global_services.rs +++ b/src/query/service/src/global_services.rs @@ -31,11 +31,14 @@ use databend_common_storage::ShareTableConfig; use databend_common_storages_hive::HiveCreator; use databend_common_storages_iceberg::IcebergCreator; use databend_common_tracing::GlobalLogger; +use databend_common_users::builtin::BuiltIn; use databend_common_users::RoleCacheManager; use databend_common_users::UserApiProvider; use databend_storages_common_cache_manager::CacheManager; use crate::auth::AuthMgr; +use crate::builtin::BuiltinUDFs; +use crate::builtin::BuiltinUsers; use crate::catalogs::DatabaseCatalog; use crate::clusters::ClusterDiscovery; use crate::locks::LockManager; @@ -107,13 +110,27 @@ impl GlobalServices { SessionManager::init(config)?; LockManager::init()?; AuthMgr::init(config)?; - UserApiProvider::init( - config.meta.to_meta_grpc_client_conf(), - config.query.idm.clone(), - &config.query.tenant_id, - config.query.tenant_quota.clone(), - ) - .await?; + + // Init user manager. + // Builtin users and udfs are created here. + { + let built_in_users = BuiltinUsers::create(config.query.builtin.users.clone()); + let built_in_udfs = BuiltinUDFs::create(config.query.builtin.udfs.clone()); + + // We will ignore the error here, and just log a error. + let builtin = BuiltIn { + users: built_in_users.to_auth_infos(), + udfs: built_in_udfs.to_udfs(), + }; + UserApiProvider::init( + config.meta.to_meta_grpc_client_conf(), + builtin, + &config.query.tenant_id, + config.query.tenant_quota.clone(), + ) + .await?; + } + RoleCacheManager::init()?; ShareEndpointManager::init()?; diff --git a/src/query/service/src/lib.rs b/src/query/service/src/lib.rs index 1ab8819c39d53..aaca587dc2f30 100644 --- a/src/query/service/src/lib.rs +++ b/src/query/service/src/lib.rs @@ -58,6 +58,7 @@ pub mod stream; pub mod table_functions; pub mod test_kits; +mod builtin; mod global_services; pub use databend_common_sql as sql; diff --git a/src/query/service/src/test_kits/config.rs b/src/query/service/src/test_kits/config.rs index 05d891b1ffbef..c75dad974e4e2 100644 --- a/src/query/service/src/test_kits/config.rs +++ b/src/query/service/src/test_kits/config.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; - use databend_common_base::base::GlobalUniqName; +use databend_common_config::BuiltInConfig; use databend_common_config::InnerConfig; -use databend_common_meta_app::principal::AuthInfo; +use databend_common_config::UDFConfig; +use databend_common_config::UserAuthConfig; +use databend_common_config::UserConfig; use databend_common_meta_app::storage::StorageFsConfig; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::tenant::Tenant; -use databend_common_users::idm_config::IDMConfig; use tempfile::TempDir; pub struct ConfigBuilder { @@ -32,10 +32,27 @@ impl ConfigBuilder { let mut conf = InnerConfig::default(); conf.query.tenant_id = Tenant::new_literal("test"); conf.log = databend_common_tracing::Config::new_testing(); - // add idm users for test - let mut users = HashMap::new(); - users.insert("root".to_string(), AuthInfo::None); - conf.query.idm = IDMConfig { users }; + + // add builtin users for test + let users = vec![UserConfig { + name: "root".to_string(), + auth: UserAuthConfig { + auth_type: "no_password".to_string(), + auth_string: None, + }, + }]; + + // add builtin udfs for test + let udfs = vec![UDFConfig { + name: "test_builtin_ping".to_string(), + definition: "CREATE OR REPLACE FUNCTION test_builtin_ping (STRING) + RETURNS STRING + LANGUAGE python +HANDLER = 'ping' +ADDRESS = 'https://databend.com';" + .to_string(), + }]; + conf.query.builtin = BuiltInConfig { users, udfs }; // set node_id to a unique value conf.query.node_id = GlobalUniqName::unique(); @@ -90,10 +107,12 @@ impl ConfigBuilder { self } - pub fn add_user(mut self, user_name: &str, auth_info: AuthInfo) -> ConfigBuilder { - let mut users = HashMap::new(); - users.insert(user_name.to_string(), auth_info); - self.conf.query.idm = IDMConfig { users }; + pub fn add_user(mut self, _user_name: &str, user: UserConfig) -> ConfigBuilder { + let users = vec![user]; + self.conf.query.builtin = BuiltInConfig { + users, + udfs: vec![], + }; self } diff --git a/src/query/service/tests/it/servers/flight_sql/flight_sql_handler.rs b/src/query/service/tests/it/servers/flight_sql/flight_sql_handler.rs index 20cbd5a37df70..630867f5dd55e 100644 --- a/src/query/service/tests/it/servers/flight_sql/flight_sql_handler.rs +++ b/src/query/service/tests/it/servers/flight_sql/flight_sql_handler.rs @@ -25,8 +25,9 @@ use arrow_schema::ArrowError; use databend_common_base::base::tokio; use databend_common_base::runtime::Runtime; use databend_common_config::InnerConfig; +use databend_common_config::UserAuthConfig; +use databend_common_config::UserConfig; use databend_common_exception::Result; -use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::PasswordHashMethod; use databend_query::servers::flight_sql::flight_sql_service::FlightSqlServiceImpl; use databend_query::test_kits::ConfigBuilder; @@ -78,12 +79,15 @@ fn prepare_config() -> InnerConfig { let hash_method = PasswordHashMethod::DoubleSha1; let hash_value = hash_method.hash(TEST_PASSWORD.as_bytes()); - let auth_info = AuthInfo::Password { - hash_value, - hash_method, + let user_config = UserConfig { + name: TEST_USER.to_string(), + auth: UserAuthConfig { + auth_type: "double_sha1_password".to_string(), + auth_string: Some(hex::encode(hash_value)), + }, }; ConfigBuilder::create() - .add_user(TEST_USER, auth_info) + .add_user(TEST_USER, user_config) .build() } diff --git a/src/query/service/tests/it/servers/http/http_query_handlers.rs b/src/query/service/tests/it/servers/http/http_query_handlers.rs index b84ed17dcb90e..ee30738784c8f 100644 --- a/src/query/service/tests/it/servers/http/http_query_handlers.rs +++ b/src/query/service/tests/it/servers/http/http_query_handlers.rs @@ -21,9 +21,10 @@ use base64::engine::general_purpose; use base64::prelude::*; use databend_common_base::base::get_free_tcp_port; use databend_common_base::base::tokio; +use databend_common_config::UserAuthConfig; +use databend_common_config::UserConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::PasswordHashMethod; use databend_common_users::CustomClaims; use databend_common_users::EnsureUser; @@ -1616,12 +1617,16 @@ async fn test_auth_configured_user() -> Result<()> { let hash_method = PasswordHashMethod::DoubleSha1; let hash_value = hash_method.hash(pass_word.as_bytes()); - let auth_info = AuthInfo::Password { - hash_value, - hash_method, + let user_config = UserConfig { + name: user_name.to_string(), + auth: UserAuthConfig { + auth_type: "double_sha1_password".to_string(), + auth_string: Some(hex::encode(hash_value)), + }, }; + let config = ConfigBuilder::create() - .add_user(user_name, auth_info) + .add_user(user_name, user_config) .build(); let _fixture = TestFixture::setup_with_config(&config).await?; @@ -1653,7 +1658,7 @@ async fn test_txn_error() -> Result<()> { { let mut session = session.clone(); session.last_server_info = None; - let json = serde_json::json! ({ + let json = serde_json::json!({ "sql": "select 1", "session": session, "pagination": {"wait_time_secs": wait_time_secs} @@ -1671,7 +1676,7 @@ async fn test_txn_error() -> Result<()> { if let Some(s) = &mut session.last_server_info { s.id = "abc".to_string() } - let json = serde_json::json! ({ + let json = serde_json::json!({ "sql": "select 1", "session": session, "pagination": {"wait_time_secs": wait_time_secs} @@ -1686,7 +1691,7 @@ async fn test_txn_error() -> Result<()> { if let Some(s) = &mut session.last_server_info { s.start_time = "abc".to_string() } - let json = serde_json::json! ({ + let json = serde_json::json!({ "sql": "select 1", "session": session, "pagination": {"wait_time_secs": wait_time_secs} @@ -1712,7 +1717,7 @@ async fn test_txn_timeout() -> Result<()> { let session = session.clone(); let last_query_id = session.last_query_ids.first().unwrap().to_string(); - let json = serde_json::json! ({ + let json = serde_json::json!({ "sql": "select 1", "session": session, "pagination": {"wait_time_secs": wait_time_secs} diff --git a/src/query/service/tests/it/storages/testdata/configs_table_basic.txt b/src/query/service/tests/it/storages/testdata/configs_table_basic.txt index ad522d70b7915..f4aca791f793c 100644 --- a/src/query/service/tests/it/storages/testdata/configs_table_basic.txt +++ b/src/query/service/tests/it/storages/testdata/configs_table_basic.txt @@ -1,186 +1,187 @@ ---------- TABLE INFO ------------ DB.Table: 'system'.'configs', Table: configs-table_id:1, ver:0, Engine: SystemConfigs -------- TABLE CONTENTS ---------- -+-----------+--------------------------------------------+----------------------------------------------------------------+----------+ -| Column 0 | Column 1 | Column 2 | Column 3 | -+-----------+--------------------------------------------+----------------------------------------------------------------+----------+ -| 'cache' | 'data_cache_key_reload_policy' | 'reset' | '' | -| 'cache' | 'data_cache_storage' | 'none' | '' | -| 'cache' | 'disk.max_bytes' | '21474836480' | '' | -| 'cache' | 'disk.path' | './.databend/_cache' | '' | -| 'cache' | 'enable_table_bloom_index_cache' | 'true' | '' | -| 'cache' | 'enable_table_meta_cache' | 'true' | '' | -| 'cache' | 'inverted_index_filter_memory_ratio' | '0' | '' | -| 'cache' | 'inverted_index_filter_size' | '2147483648' | '' | -| 'cache' | 'inverted_index_meta_count' | '3000' | '' | -| 'cache' | 'table_bloom_index_filter_count' | '0' | '' | -| 'cache' | 'table_bloom_index_filter_size' | '2147483648' | '' | -| 'cache' | 'table_bloom_index_meta_count' | '3000' | '' | -| 'cache' | 'table_data_cache_population_queue_size' | '0' | '' | -| 'cache' | 'table_data_deserialized_data_bytes' | '0' | '' | -| 'cache' | 'table_data_deserialized_memory_ratio' | '0' | '' | -| 'cache' | 'table_meta_segment_bytes' | '1073741824' | '' | -| 'cache' | 'table_meta_segment_count' | 'null' | '' | -| 'cache' | 'table_meta_snapshot_count' | '256' | '' | -| 'cache' | 'table_meta_statistic_count' | '256' | '' | -| 'cache' | 'table_prune_partitions_count' | '256' | '' | -| 'log' | 'dir' | './.databend/logs' | '' | -| 'log' | 'file.dir' | './.databend/logs' | '' | -| 'log' | 'file.format' | 'text' | '' | -| 'log' | 'file.level' | 'DEBUG' | '' | -| 'log' | 'file.limit' | '48' | '' | -| 'log' | 'file.on' | 'true' | '' | -| 'log' | 'file.prefix_filter' | 'databend_,openraft' | '' | -| 'log' | 'level' | 'DEBUG' | '' | -| 'log' | 'log_dir' | 'null' | '' | -| 'log' | 'log_level' | 'null' | '' | -| 'log' | 'log_query_enabled' | 'null' | '' | -| 'log' | 'otlp.level' | 'INFO' | '' | -| 'log' | 'otlp.on' | 'false' | '' | -| 'log' | 'otlp.otlp_endpoint' | 'http://127.0.0.1:4317' | '' | -| 'log' | 'otlp.otlp_protocol' | 'grpc' | '' | -| 'log' | 'profile.dir' | '' | '' | -| 'log' | 'profile.on' | 'false' | '' | -| 'log' | 'query.dir' | '' | '' | -| 'log' | 'query.on' | 'false' | '' | -| 'log' | 'query_enabled' | 'null' | '' | -| 'log' | 'stderr.format' | 'text' | '' | -| 'log' | 'stderr.level' | 'WARN' | '' | -| 'log' | 'stderr.on' | 'true' | '' | -| 'log' | 'structlog.dir' | '' | '' | -| 'log' | 'structlog.on' | 'false' | '' | -| 'log' | 'tracing.capture_log_level' | 'INFO' | '' | -| 'log' | 'tracing.on' | 'false' | '' | -| 'log' | 'tracing.otlp_endpoint' | 'http://127.0.0.1:4317' | '' | -| 'log' | 'tracing.otlp_protocol' | 'grpc' | '' | -| 'meta' | 'auto_sync_interval' | '0' | '' | -| 'meta' | 'client_timeout_in_second' | '10' | '' | -| 'meta' | 'embedded_dir' | '' | '' | -| 'meta' | 'endpoints' | '' | '' | -| 'meta' | 'meta_client_timeout_in_second' | 'null' | '' | -| 'meta' | 'meta_embedded_dir' | 'null' | '' | -| 'meta' | 'meta_password' | 'null' | '' | -| 'meta' | 'meta_username' | 'null' | '' | -| 'meta' | 'password' | '' | '' | -| 'meta' | 'rpc_tls_meta_server_root_ca_cert' | '' | '' | -| 'meta' | 'rpc_tls_meta_service_domain_name' | 'localhost' | '' | -| 'meta' | 'unhealth_endpoint_evict_time' | '120' | '' | -| 'meta' | 'username' | 'root' | '' | -| 'query' | 'admin_api_address' | '127.0.0.1:8080' | '' | -| 'query' | 'api_tls_server_cert' | '' | '' | -| 'query' | 'api_tls_server_key' | '' | '' | -| 'query' | 'api_tls_server_root_ca_cert' | '' | '' | -| 'query' | 'clickhouse_handler_host' | '127.0.0.1' | '' | -| 'query' | 'clickhouse_handler_port' | '9000' | '' | -| 'query' | 'clickhouse_http_handler_host' | '127.0.0.1' | '' | -| 'query' | 'clickhouse_http_handler_port' | '8124' | '' | -| 'query' | 'cloud_control_grpc_server_address' | 'null' | '' | -| 'query' | 'cloud_control_grpc_timeout' | '0' | '' | -| 'query' | 'cluster_id' | '' | '' | -| 'query' | 'data_retention_time_in_days_max' | '90' | '' | -| 'query' | 'databend_enterprise_license' | 'null' | '' | -| 'query' | 'default_compression' | 'auto' | '' | -| 'query' | 'default_storage_format' | 'auto' | '' | -| 'query' | 'disable_system_table_load' | 'false' | '' | -| 'query' | 'enable_udf_server' | 'false' | '' | -| 'query' | 'flight_api_address' | '127.0.0.1:9090' | '' | -| 'query' | 'flight_sql_handler_host' | '127.0.0.1' | '' | -| 'query' | 'flight_sql_handler_port' | '8900' | '' | -| 'query' | 'flight_sql_tls_server_cert' | '' | '' | -| 'query' | 'flight_sql_tls_server_key' | '' | '' | -| 'query' | 'http_handler_host' | '127.0.0.1' | '' | -| 'query' | 'http_handler_port' | '8000' | '' | -| 'query' | 'http_handler_result_timeout_secs' | '60' | '' | -| 'query' | 'http_handler_tls_server_cert' | '' | '' | -| 'query' | 'http_handler_tls_server_key' | '' | '' | -| 'query' | 'http_handler_tls_server_root_ca_cert' | '' | '' | -| 'query' | 'internal_enable_sandbox_tenant' | 'false' | '' | -| 'query' | 'internal_merge_on_read_mutation' | 'false' | '' | -| 'query' | 'jwt_key_file' | '' | '' | -| 'query' | 'jwt_key_files' | '' | '' | -| 'query' | 'management_mode' | 'false' | '' | -| 'query' | 'max_active_sessions' | '256' | '' | -| 'query' | 'max_memory_limit_enabled' | 'false' | '' | -| 'query' | 'max_query_log_size' | '10000' | '' | -| 'query' | 'max_running_queries' | '8' | '' | -| 'query' | 'max_server_memory_usage' | '0' | '' | -| 'query' | 'max_storage_io_requests' | 'null' | '' | -| 'query' | 'metric_api_address' | '127.0.0.1:7070' | '' | -| 'query' | 'mysql_handler_host' | '127.0.0.1' | '' | -| 'query' | 'mysql_handler_port' | '3307' | '' | -| 'query' | 'mysql_handler_tcp_keepalive_timeout_secs' | '120' | '' | -| 'query' | 'mysql_tls_server_cert' | '' | '' | -| 'query' | 'mysql_tls_server_key' | '' | '' | -| 'query' | 'num_cpus' | '0' | '' | -| 'query' | 'openai_api_chat_base_url' | 'https://api.openai.com/v1/' | '' | -| 'query' | 'openai_api_completion_model' | 'gpt-3.5-turbo' | '' | -| 'query' | 'openai_api_embedding_base_url' | 'https://api.openai.com/v1/' | '' | -| 'query' | 'openai_api_embedding_model' | 'text-embedding-ada-002' | '' | -| 'query' | 'openai_api_key' | '' | '' | -| 'query' | 'openai_api_version' | '' | '' | -| 'query' | 'parquet_fast_read_bytes' | 'null' | '' | -| 'query' | 'quota' | 'null' | '' | -| 'query' | 'rpc_client_timeout_secs' | '0' | '' | -| 'query' | 'rpc_tls_query_server_root_ca_cert' | '' | '' | -| 'query' | 'rpc_tls_query_service_domain_name' | 'localhost' | '' | -| 'query' | 'rpc_tls_server_cert' | '' | '' | -| 'query' | 'rpc_tls_server_key' | '' | '' | -| 'query' | 'share_endpoint_address' | '' | '' | -| 'query' | 'share_endpoint_auth_token_file' | '' | '' | -| 'query' | 'shutdown_wait_timeout_ms' | '5000' | '' | -| 'query' | 'table_engine_memory_enabled' | 'true' | '' | -| 'query' | 'tenant_id' | 'test' | '' | -| 'query' | 'udf_server_allow_list' | '' | '' | -| 'query' | 'users' | '{"name":"root","auth_type":"no_password","auth_string":null}' | '' | -| 'storage' | 'allow_insecure' | 'true' | '' | -| 'storage' | 'azblob.account_key' | '' | '' | -| 'storage' | 'azblob.account_name' | '' | '' | -| 'storage' | 'azblob.container' | '' | '' | -| 'storage' | 'azblob.endpoint_url' | '' | '' | -| 'storage' | 'azblob.root' | '' | '' | -| 'storage' | 'cos.bucket' | '' | '' | -| 'storage' | 'cos.endpoint_url' | '' | '' | -| 'storage' | 'cos.root' | '' | '' | -| 'storage' | 'cos.secret_id' | '' | '' | -| 'storage' | 'cos.secret_key' | '' | '' | -| 'storage' | 'fs.data_path' | '_data' | '' | -| 'storage' | 'gcs.bucket' | '' | '' | -| 'storage' | 'gcs.credential' | '' | '' | -| 'storage' | 'gcs.endpoint_url' | 'https://storage.googleapis.com' | '' | -| 'storage' | 'gcs.root' | '' | '' | -| 'storage' | 'hdfs.name_node' | '' | '' | -| 'storage' | 'hdfs.root' | '' | '' | -| 'storage' | 'num_cpus' | '0' | '' | -| 'storage' | 'obs.access_key_id' | '' | '' | -| 'storage' | 'obs.bucket' | '' | '' | -| 'storage' | 'obs.endpoint_url' | '' | '' | -| 'storage' | 'obs.root' | '' | '' | -| 'storage' | 'obs.secret_access_key' | '' | '' | -| 'storage' | 'oss.access_key_id' | '' | '' | -| 'storage' | 'oss.access_key_secret' | '' | '' | -| 'storage' | 'oss.bucket' | '' | '' | -| 'storage' | 'oss.endpoint_url' | '' | '' | -| 'storage' | 'oss.presign_endpoint_url' | '' | '' | -| 'storage' | 'oss.root' | '' | '' | -| 'storage' | 'oss.server_side_encryption' | '' | '' | -| 'storage' | 'oss.server_side_encryption_key_id' | '' | '' | -| 'storage' | 's3.access_key_id' | '' | '' | -| 'storage' | 's3.bucket' | '' | '' | -| 'storage' | 's3.enable_virtual_host_style' | 'false' | '' | -| 'storage' | 's3.endpoint_url' | 'https://s3.amazonaws.com' | '' | -| 'storage' | 's3.external_id' | '' | '' | -| 'storage' | 's3.master_key' | '' | '' | -| 'storage' | 's3.region' | '' | '' | -| 'storage' | 's3.role_arn' | '' | '' | -| 'storage' | 's3.root' | '' | '' | -| 'storage' | 's3.secret_access_key' | '' | '' | -| 'storage' | 's3.security_token' | '' | '' | -| 'storage' | 'storage_num_cpus' | 'null' | '' | -| 'storage' | 'storage_type' | 'null' | '' | -| 'storage' | 'type' | 'fs' | '' | -| 'storage' | 'webhdfs.delegation' | '' | '' | -| 'storage' | 'webhdfs.endpoint_url' | '' | '' | -| 'storage' | 'webhdfs.root' | '' | '' | -+-----------+--------------------------------------------+----------------------------------------------------------------+----------+ ++-----------+--------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+ +| Column 0 | Column 1 | Column 2 | Column 3 | ++-----------+--------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+ +| 'cache' | 'data_cache_key_reload_policy' | 'reset' | '' | +| 'cache' | 'data_cache_storage' | 'none' | '' | +| 'cache' | 'disk.max_bytes' | '21474836480' | '' | +| 'cache' | 'disk.path' | './.databend/_cache' | '' | +| 'cache' | 'enable_table_bloom_index_cache' | 'true' | '' | +| 'cache' | 'enable_table_meta_cache' | 'true' | '' | +| 'cache' | 'inverted_index_filter_memory_ratio' | '0' | '' | +| 'cache' | 'inverted_index_filter_size' | '2147483648' | '' | +| 'cache' | 'inverted_index_meta_count' | '3000' | '' | +| 'cache' | 'table_bloom_index_filter_count' | '0' | '' | +| 'cache' | 'table_bloom_index_filter_size' | '2147483648' | '' | +| 'cache' | 'table_bloom_index_meta_count' | '3000' | '' | +| 'cache' | 'table_data_cache_population_queue_size' | '0' | '' | +| 'cache' | 'table_data_deserialized_data_bytes' | '0' | '' | +| 'cache' | 'table_data_deserialized_memory_ratio' | '0' | '' | +| 'cache' | 'table_meta_segment_bytes' | '1073741824' | '' | +| 'cache' | 'table_meta_segment_count' | 'null' | '' | +| 'cache' | 'table_meta_snapshot_count' | '256' | '' | +| 'cache' | 'table_meta_statistic_count' | '256' | '' | +| 'cache' | 'table_prune_partitions_count' | '256' | '' | +| 'log' | 'dir' | './.databend/logs' | '' | +| 'log' | 'file.dir' | './.databend/logs' | '' | +| 'log' | 'file.format' | 'text' | '' | +| 'log' | 'file.level' | 'DEBUG' | '' | +| 'log' | 'file.limit' | '48' | '' | +| 'log' | 'file.on' | 'true' | '' | +| 'log' | 'file.prefix_filter' | 'databend_,openraft' | '' | +| 'log' | 'level' | 'DEBUG' | '' | +| 'log' | 'log_dir' | 'null' | '' | +| 'log' | 'log_level' | 'null' | '' | +| 'log' | 'log_query_enabled' | 'null' | '' | +| 'log' | 'otlp.level' | 'INFO' | '' | +| 'log' | 'otlp.on' | 'false' | '' | +| 'log' | 'otlp.otlp_endpoint' | 'http://127.0.0.1:4317' | '' | +| 'log' | 'otlp.otlp_protocol' | 'grpc' | '' | +| 'log' | 'profile.dir' | '' | '' | +| 'log' | 'profile.on' | 'false' | '' | +| 'log' | 'query.dir' | '' | '' | +| 'log' | 'query.on' | 'false' | '' | +| 'log' | 'query_enabled' | 'null' | '' | +| 'log' | 'stderr.format' | 'text' | '' | +| 'log' | 'stderr.level' | 'WARN' | '' | +| 'log' | 'stderr.on' | 'true' | '' | +| 'log' | 'structlog.dir' | '' | '' | +| 'log' | 'structlog.on' | 'false' | '' | +| 'log' | 'tracing.capture_log_level' | 'INFO' | '' | +| 'log' | 'tracing.on' | 'false' | '' | +| 'log' | 'tracing.otlp_endpoint' | 'http://127.0.0.1:4317' | '' | +| 'log' | 'tracing.otlp_protocol' | 'grpc' | '' | +| 'meta' | 'auto_sync_interval' | '0' | '' | +| 'meta' | 'client_timeout_in_second' | '10' | '' | +| 'meta' | 'embedded_dir' | '' | '' | +| 'meta' | 'endpoints' | '' | '' | +| 'meta' | 'meta_client_timeout_in_second' | 'null' | '' | +| 'meta' | 'meta_embedded_dir' | 'null' | '' | +| 'meta' | 'meta_password' | 'null' | '' | +| 'meta' | 'meta_username' | 'null' | '' | +| 'meta' | 'password' | '' | '' | +| 'meta' | 'rpc_tls_meta_server_root_ca_cert' | '' | '' | +| 'meta' | 'rpc_tls_meta_service_domain_name' | 'localhost' | '' | +| 'meta' | 'unhealth_endpoint_evict_time' | '120' | '' | +| 'meta' | 'username' | 'root' | '' | +| 'query' | 'admin_api_address' | '127.0.0.1:8080' | '' | +| 'query' | 'api_tls_server_cert' | '' | '' | +| 'query' | 'api_tls_server_key' | '' | '' | +| 'query' | 'api_tls_server_root_ca_cert' | '' | '' | +| 'query' | 'clickhouse_handler_host' | '127.0.0.1' | '' | +| 'query' | 'clickhouse_handler_port' | '9000' | '' | +| 'query' | 'clickhouse_http_handler_host' | '127.0.0.1' | '' | +| 'query' | 'clickhouse_http_handler_port' | '8124' | '' | +| 'query' | 'cloud_control_grpc_server_address' | 'null' | '' | +| 'query' | 'cloud_control_grpc_timeout' | '0' | '' | +| 'query' | 'cluster_id' | '' | '' | +| 'query' | 'data_retention_time_in_days_max' | '90' | '' | +| 'query' | 'databend_enterprise_license' | 'null' | '' | +| 'query' | 'default_compression' | 'auto' | '' | +| 'query' | 'default_storage_format' | 'auto' | '' | +| 'query' | 'disable_system_table_load' | 'false' | '' | +| 'query' | 'enable_udf_server' | 'false' | '' | +| 'query' | 'flight_api_address' | '127.0.0.1:9090' | '' | +| 'query' | 'flight_sql_handler_host' | '127.0.0.1' | '' | +| 'query' | 'flight_sql_handler_port' | '8900' | '' | +| 'query' | 'flight_sql_tls_server_cert' | '' | '' | +| 'query' | 'flight_sql_tls_server_key' | '' | '' | +| 'query' | 'http_handler_host' | '127.0.0.1' | '' | +| 'query' | 'http_handler_port' | '8000' | '' | +| 'query' | 'http_handler_result_timeout_secs' | '60' | '' | +| 'query' | 'http_handler_tls_server_cert' | '' | '' | +| 'query' | 'http_handler_tls_server_key' | '' | '' | +| 'query' | 'http_handler_tls_server_root_ca_cert' | '' | '' | +| 'query' | 'internal_enable_sandbox_tenant' | 'false' | '' | +| 'query' | 'internal_merge_on_read_mutation' | 'false' | '' | +| 'query' | 'jwt_key_file' | '' | '' | +| 'query' | 'jwt_key_files' | '' | '' | +| 'query' | 'management_mode' | 'false' | '' | +| 'query' | 'max_active_sessions' | '256' | '' | +| 'query' | 'max_memory_limit_enabled' | 'false' | '' | +| 'query' | 'max_query_log_size' | '10000' | '' | +| 'query' | 'max_running_queries' | '8' | '' | +| 'query' | 'max_server_memory_usage' | '0' | '' | +| 'query' | 'max_storage_io_requests' | 'null' | '' | +| 'query' | 'metric_api_address' | '127.0.0.1:7070' | '' | +| 'query' | 'mysql_handler_host' | '127.0.0.1' | '' | +| 'query' | 'mysql_handler_port' | '3307' | '' | +| 'query' | 'mysql_handler_tcp_keepalive_timeout_secs' | '120' | '' | +| 'query' | 'mysql_tls_server_cert' | '' | '' | +| 'query' | 'mysql_tls_server_key' | '' | '' | +| 'query' | 'num_cpus' | '0' | '' | +| 'query' | 'openai_api_chat_base_url' | 'https://api.openai.com/v1/' | '' | +| 'query' | 'openai_api_completion_model' | 'gpt-3.5-turbo' | '' | +| 'query' | 'openai_api_embedding_base_url' | 'https://api.openai.com/v1/' | '' | +| 'query' | 'openai_api_embedding_model' | 'text-embedding-ada-002' | '' | +| 'query' | 'openai_api_key' | '' | '' | +| 'query' | 'openai_api_version' | '' | '' | +| 'query' | 'parquet_fast_read_bytes' | 'null' | '' | +| 'query' | 'quota' | 'null' | '' | +| 'query' | 'rpc_client_timeout_secs' | '0' | '' | +| 'query' | 'rpc_tls_query_server_root_ca_cert' | '' | '' | +| 'query' | 'rpc_tls_query_service_domain_name' | 'localhost' | '' | +| 'query' | 'rpc_tls_server_cert' | '' | '' | +| 'query' | 'rpc_tls_server_key' | '' | '' | +| 'query' | 'share_endpoint_address' | '' | '' | +| 'query' | 'share_endpoint_auth_token_file' | '' | '' | +| 'query' | 'shutdown_wait_timeout_ms' | '5000' | '' | +| 'query' | 'table_engine_memory_enabled' | 'true' | '' | +| 'query' | 'tenant_id' | 'test' | '' | +| 'query' | 'udf_server_allow_list' | '' | '' | +| 'query' | 'udfs' | '{"name":"test_builtin_ping","definition":"CREATE OR REPLACE FUNCTION test_builtin_ping (STRING)\n RETURNS STRING\n LANGUAGE python\nHANDLER = 'ping'\nADDRESS = 'https://databend.com';"}' | '' | +| 'query' | 'users' | '{"name":"root","auth_type":"no_password","auth_string":null}' | '' | +| 'storage' | 'allow_insecure' | 'true' | '' | +| 'storage' | 'azblob.account_key' | '' | '' | +| 'storage' | 'azblob.account_name' | '' | '' | +| 'storage' | 'azblob.container' | '' | '' | +| 'storage' | 'azblob.endpoint_url' | '' | '' | +| 'storage' | 'azblob.root' | '' | '' | +| 'storage' | 'cos.bucket' | '' | '' | +| 'storage' | 'cos.endpoint_url' | '' | '' | +| 'storage' | 'cos.root' | '' | '' | +| 'storage' | 'cos.secret_id' | '' | '' | +| 'storage' | 'cos.secret_key' | '' | '' | +| 'storage' | 'fs.data_path' | '_data' | '' | +| 'storage' | 'gcs.bucket' | '' | '' | +| 'storage' | 'gcs.credential' | '' | '' | +| 'storage' | 'gcs.endpoint_url' | 'https://storage.googleapis.com' | '' | +| 'storage' | 'gcs.root' | '' | '' | +| 'storage' | 'hdfs.name_node' | '' | '' | +| 'storage' | 'hdfs.root' | '' | '' | +| 'storage' | 'num_cpus' | '0' | '' | +| 'storage' | 'obs.access_key_id' | '' | '' | +| 'storage' | 'obs.bucket' | '' | '' | +| 'storage' | 'obs.endpoint_url' | '' | '' | +| 'storage' | 'obs.root' | '' | '' | +| 'storage' | 'obs.secret_access_key' | '' | '' | +| 'storage' | 'oss.access_key_id' | '' | '' | +| 'storage' | 'oss.access_key_secret' | '' | '' | +| 'storage' | 'oss.bucket' | '' | '' | +| 'storage' | 'oss.endpoint_url' | '' | '' | +| 'storage' | 'oss.presign_endpoint_url' | '' | '' | +| 'storage' | 'oss.root' | '' | '' | +| 'storage' | 'oss.server_side_encryption' | '' | '' | +| 'storage' | 'oss.server_side_encryption_key_id' | '' | '' | +| 'storage' | 's3.access_key_id' | '' | '' | +| 'storage' | 's3.bucket' | '' | '' | +| 'storage' | 's3.enable_virtual_host_style' | 'false' | '' | +| 'storage' | 's3.endpoint_url' | 'https://s3.amazonaws.com' | '' | +| 'storage' | 's3.external_id' | '' | '' | +| 'storage' | 's3.master_key' | '' | '' | +| 'storage' | 's3.region' | '' | '' | +| 'storage' | 's3.role_arn' | '' | '' | +| 'storage' | 's3.root' | '' | '' | +| 'storage' | 's3.secret_access_key' | '' | '' | +| 'storage' | 's3.security_token' | '' | '' | +| 'storage' | 'storage_num_cpus' | 'null' | '' | +| 'storage' | 'storage_type' | 'null' | '' | +| 'storage' | 'type' | 'fs' | '' | +| 'storage' | 'webhdfs.delegation' | '' | '' | +| 'storage' | 'webhdfs.endpoint_url' | '' | '' | +| 'storage' | 'webhdfs.root' | '' | '' | ++-----------+--------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+ diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index 32b49fe30066b..45e5bd5e0813a 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -773,7 +773,7 @@ impl<'a> TypeChecker<'a> { "no function matches the given name: '{func_name}', do you mean {}?", possible_funcs.join(", ") )) - .set_span(*span)); + .set_span(*span)); } } } @@ -857,7 +857,7 @@ impl<'a> TypeChecker<'a> { ErrorCode::SemanticError(format!( "invalid parameter {param} for aggregate function, expected constant", )) - .set_span(*span) + .set_span(*span) })? .1; new_params.push(constant); @@ -918,7 +918,7 @@ impl<'a> TypeChecker<'a> { ErrorCode::SemanticError(format!( "invalid parameter {param} for scalar function, expected constant", )) - .set_span(*span) + .set_span(*span) })? .1; new_params.push(constant); @@ -1834,13 +1834,13 @@ impl<'a> TypeChecker<'a> { "incorrect number of parameters in lambda function, {} expects 1 parameter, but got {}", func_name, params.len() )) - .set_span(span)); + .set_span(span)); } else if func_name == "array_reduce" && params.len() != 2 { return Err(ErrorCode::SemanticError(format!( "incorrect number of parameters in lambda function, {} expects 2 parameters, but got {}", func_name, params.len() )) - .set_span(span)); + .set_span(span)); } if args.len() != 1 { @@ -1888,7 +1888,7 @@ impl<'a> TypeChecker<'a> { return Err(ErrorCode::SemanticError( "invalid lambda function for `array_filter`, the result data type of lambda function must be boolean".to_string() ) - .set_span(span)); + .set_span(span)); } } else if func_name == "array_reduce" { // transform arg type @@ -2053,7 +2053,7 @@ impl<'a> TypeChecker<'a> { "invalid arguments for search function, field must be a column or constant string, but got {}", constant_expr.value )) - .set_span(constant_expr.span)); + .set_span(constant_expr.span)); }; // fields are separated by commas and boost is separated by ^ @@ -2066,7 +2066,7 @@ impl<'a> TypeChecker<'a> { "invalid arguments for search function, field string must have only one boost, but got {}", constant_field )) - .set_span(constant_expr.span)); + .set_span(constant_expr.span)); } let column_expr = Expr::ColumnRef { span: constant_expr.span, @@ -2095,7 +2095,7 @@ impl<'a> TypeChecker<'a> { "invalid arguments for search function, boost must be a float value, but got {}", field_boosts[1] )) - .set_span(constant_expr.span)); + .set_span(constant_expr.span)); } } } else { @@ -2109,7 +2109,7 @@ impl<'a> TypeChecker<'a> { return Err(ErrorCode::SemanticError( "invalid arguments for search function, field must be a column or constant string".to_string(), ) - .set_span(span)); + .set_span(span)); } }; @@ -2119,14 +2119,14 @@ impl<'a> TypeChecker<'a> { "invalid arguments for search function, query text must be a constant string, but got {}", query_arg )) - .set_span(query_scalar.span())); + .set_span(query_scalar.span())); }; let Some(query_text) = query_expr.value.as_string() else { return Err(ErrorCode::SemanticError(format!( "invalid arguments for search function, query text must be a constant string, but got {}", query_arg )) - .set_span(query_scalar.span())); + .set_span(query_scalar.span())); }; // match function didn't support query syntax, @@ -2182,14 +2182,14 @@ impl<'a> TypeChecker<'a> { "invalid arguments for search function, query text must be a constant string, but got {}", query_arg )) - .set_span(query_scalar.span())); + .set_span(query_scalar.span())); }; let Some(query_text) = query_expr.value.as_string() else { return Err(ErrorCode::SemanticError(format!( "invalid arguments for search function, query text must be a constant string, but got {}", query_arg )) - .set_span(query_scalar.span())); + .set_span(query_scalar.span())); }; let field_strs: Vec<&str> = query_text.split(' ').collect(); @@ -3279,9 +3279,9 @@ impl<'a> TypeChecker<'a> { return Ok(None); } - let udf = databend_common_base::runtime::block_on( - UserApiProvider::instance().get_udf(&self.ctx.get_tenant(), udf_name), - )?; + let udf = databend_common_base::runtime::block_on({ + UserApiProvider::instance().get_udf(&self.ctx.get_tenant(), udf_name) + })?; let Some(udf) = udf else { return Ok(None); diff --git a/src/query/users/src/idm_config.rs b/src/query/users/src/builtin.rs similarity index 85% rename from src/query/users/src/idm_config.rs rename to src/query/users/src/builtin.rs index 0eac019f7181e..6290ee6d91030 100644 --- a/src/query/users/src/idm_config.rs +++ b/src/query/users/src/builtin.rs @@ -15,8 +15,10 @@ use std::collections::HashMap; use databend_common_meta_app::principal::AuthInfo; +use databend_common_meta_app::principal::UserDefinedFunction; #[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct IDMConfig { +pub struct BuiltIn { pub users: HashMap, + pub udfs: HashMap, } diff --git a/src/query/users/src/lib.rs b/src/query/users/src/lib.rs index a7eea7b7f5a72..76518d04819ad 100644 --- a/src/query/users/src/lib.rs +++ b/src/query/users/src/lib.rs @@ -29,9 +29,9 @@ mod user_stage; mod user_udf; mod visibility_checker; +pub mod builtin; pub mod connection; pub mod file_format; -pub mod idm_config; pub mod role_cache_mgr; pub mod role_util; diff --git a/src/query/users/src/user_api.rs b/src/query/users/src/user_api.rs index 2beef30ee61b4..e372cda2ff9f0 100644 --- a/src/query/users/src/user_api.rs +++ b/src/query/users/src/user_api.rs @@ -35,6 +35,7 @@ use databend_common_management::UserApi; use databend_common_management::UserMgr; use databend_common_meta_app::principal::AuthInfo; use databend_common_meta_app::principal::RoleInfo; +use databend_common_meta_app::principal::UserDefinedFunction; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_app::tenant::TenantQuota; use databend_common_meta_kvapi::kvapi; @@ -43,24 +44,24 @@ use databend_common_meta_store::MetaStoreProvider; use databend_common_meta_types::MatchSeq; use databend_common_meta_types::MetaError; -use crate::idm_config::IDMConfig; +use crate::builtin::BuiltIn; use crate::BUILTIN_ROLE_PUBLIC; pub struct UserApiProvider { meta: MetaStore, client: Arc + Send + Sync>, - idm_config: IDMConfig, + builtin: BuiltIn, } impl UserApiProvider { #[async_backtrace::framed] pub async fn init( conf: RpcClientConf, - idm_config: IDMConfig, + builtin: BuiltIn, tenant: &Tenant, quota: Option, ) -> Result<()> { - GlobalInstance::set(Self::try_create(conf, idm_config, tenant).await?); + GlobalInstance::set(Self::try_create(conf, builtin, tenant).await?); let user_mgr = UserApiProvider::instance(); if let Some(q) = quota { let i = user_mgr.tenant_quota_api(tenant); @@ -73,14 +74,14 @@ impl UserApiProvider { #[async_backtrace::framed] pub async fn try_create( conf: RpcClientConf, - idm_config: IDMConfig, + builtin: BuiltIn, tenant: &Tenant, ) -> Result> { let client = MetaStoreProvider::new(conf).create_meta_store().await?; let user_mgr = UserApiProvider { meta: client.clone(), client: client.arc(), - idm_config, + builtin, }; // init built-in role @@ -106,7 +107,7 @@ impl UserApiProvider { conf: RpcClientConf, tenant: &Tenant, ) -> Result> { - Self::try_create(conf, IDMConfig::default(), tenant).await + Self::try_create(conf, BuiltIn::default(), tenant).await } pub fn instance() -> Arc { @@ -160,10 +161,18 @@ impl UserApiProvider { } pub fn get_configured_user(&self, user_name: &str) -> Option<&AuthInfo> { - self.idm_config.users.get(user_name) + self.builtin.users.get(user_name) } pub fn get_configured_users(&self) -> HashMap { - self.idm_config.users.clone() + self.builtin.users.clone() + } + + pub fn get_configured_udf(&self, udf_name: &str) -> Option { + self.builtin.udfs.get(udf_name).cloned() + } + + pub fn get_configured_udfs(&self) -> HashMap { + self.builtin.udfs.clone() } } diff --git a/src/query/users/src/user_mgr.rs b/src/query/users/src/user_mgr.rs index fa75c592572bf..df79a9bcf51a0 100644 --- a/src/query/users/src/user_mgr.rs +++ b/src/query/users/src/user_mgr.rs @@ -177,7 +177,7 @@ impl UserApiProvider { ) -> Result> { if self.get_configured_user(&user.username).is_some() { return Err(ErrorCode::UserAlreadyExists(format!( - "Cannot grant privileges to configured user `{}`", + "Cannot grant privileges to built-in user `{}`", user.username ))); } @@ -201,7 +201,7 @@ impl UserApiProvider { ) -> Result> { if self.get_configured_user(&user.username).is_some() { return Err(ErrorCode::UserAlreadyExists(format!( - "Cannot revoke privileges from configured user `{}`", + "Cannot revoke privileges from built-in user `{}`", user.username ))); } @@ -224,7 +224,7 @@ impl UserApiProvider { ) -> Result> { if self.get_configured_user(&user.username).is_some() { return Err(ErrorCode::UserAlreadyExists(format!( - "Cannot grant role to configured user `{}`", + "Cannot grant role to built-in user `{}`", user.username ))); } @@ -247,7 +247,7 @@ impl UserApiProvider { ) -> Result> { if self.get_configured_user(&user.username).is_some() { return Err(ErrorCode::UserAlreadyExists(format!( - "Cannot revoke role from configured user `{}`", + "Cannot revoke role from built-in user `{}`", user.username ))); } @@ -271,7 +271,7 @@ impl UserApiProvider { ) -> Result<()> { if self.get_configured_user(&user.username).is_some() { return Err(ErrorCode::UserAlreadyExists(format!( - "Configured user `{}` cannot be dropped", + "Built-in user `{}` cannot be dropped", user.username ))); } @@ -318,7 +318,7 @@ impl UserApiProvider { } if self.get_configured_user(&user.username).is_some() { return Err(ErrorCode::UserAlreadyExists(format!( - "Configured user `{}` cannot be updated", + "Built-in user `{}` cannot be updated", user.username ))); } diff --git a/src/query/users/src/user_udf.rs b/src/query/users/src/user_udf.rs index 0f357fb21c057..50e2be5b09d5f 100644 --- a/src/query/users/src/user_udf.rs +++ b/src/query/users/src/user_udf.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_management::udf::UdfApiError; use databend_common_management::udf::UdfError; @@ -32,6 +33,13 @@ impl UserApiProvider { info: UserDefinedFunction, create_option: &CreateOption, ) -> Result<()> { + if self.get_configured_udf(&info.name).is_some() { + return Err(ErrorCode::UdfAlreadyExists(format!( + "Built-in UDF `{}` already exists", + info.name + ))); + } + let udf_api = self.udf_api(tenant); udf_api.add_udf(info, create_option).await??; Ok(()) @@ -40,6 +48,13 @@ impl UserApiProvider { // Update a UDF. #[async_backtrace::framed] pub async fn update_udf(&self, tenant: &Tenant, info: UserDefinedFunction) -> Result { + if self.get_configured_udf(&info.name).is_some() { + return Err(ErrorCode::UserAlreadyExists(format!( + "Built-in UDF `{}` cannot be updated", + info.name + ))); + } + let res = self .udf_api(tenant) .update_udf(info, MatchSeq::GE(1)) @@ -56,8 +71,12 @@ impl UserApiProvider { tenant: &Tenant, udf_name: &str, ) -> Result, UdfApiError> { - let seqv = self.udf_api(tenant).get_udf(udf_name).await?; - Ok(seqv.map(|x| x.data)) + if let Some(udf) = self.get_configured_udf(udf_name) { + Ok(Some(udf)) + } else { + let seqv = self.udf_api(tenant).get_udf(udf_name).await?; + Ok(seqv.map(|x| x.data)) + } } #[async_backtrace::framed] @@ -87,6 +106,14 @@ impl UserApiProvider { udf_name: &str, allow_no_change: bool, ) -> std::result::Result, UdfApiError> { + if self.get_configured_udf(udf_name).is_some() { + return Ok(Err(UdfError::Exists { + tenant: tenant.tenant_name().to_string(), + name: udf_name.to_string(), + reason: "Built-in UDF cannot be dropped".to_string(), + })); + } + let dropped = self .udf_api(tenant) .drop_udf(udf_name, MatchSeq::GE(1)) diff --git a/tests/sqllogictests/suites/udf_server/udf_builtin.test b/tests/sqllogictests/suites/udf_server/udf_builtin.test new file mode 100644 index 0000000000000..6aa87e214123a --- /dev/null +++ b/tests/sqllogictests/suites/udf_server/udf_builtin.test @@ -0,0 +1,14 @@ +# Please start the UDF Server first before running this test: +# python3 tests/udf/udf_server.py +# + +query T +select ping('test'); +---- +test + +statement error 2603 +CREATE FUNCTION ping(STRING) RETURNS STRING LANGUAGE python HANDLER = 'ping' ADDRESS = 'http://0.0.0.0:8815'; + +statement error 2603 +DROP FUNCTION IF EXISTS ping; diff --git a/tests/udf/udf_server.py b/tests/udf/udf_server.py index 0472cb9d89742..ac657ced71d9a 100644 --- a/tests/udf/udf_server.py +++ b/tests/udf/udf_server.py @@ -21,7 +21,6 @@ # https://github.com/datafuselabs/databend-udf from databend_udf import udf, UDFServer - logging.basicConfig(level=logging.INFO) @@ -163,7 +162,7 @@ def json_concat(list: List[Any]) -> Any: result_type="TUPLE(VARIANT NULL, VARIANT NULL)", ) def tuple_access( - tup: Tuple[List[Any], int, str], idx1: int, idx2: int + tup: Tuple[List[Any], int, str], idx1: int, idx2: int ) -> Tuple[Any, Any]: v1 = None if idx1 == 0 or idx1 > len(tup) else tup[idx1 - 1] v2 = None if idx2 == 0 or idx2 > len(tup) else tup[idx2 - 1] @@ -194,21 +193,21 @@ def tuple_access( result_type=f"TUPLE({','.join(f'{t} NULL' for t in ALL_SCALAR_TYPES)})", ) def return_all( - bool, - i8, - i16, - i32, - i64, - u8, - u16, - u32, - u64, - f32, - f64, - date, - timestamp, - varchar, - json, + bool, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f32, + f64, + date, + timestamp, + varchar, + json, ): return ( bool, @@ -234,21 +233,21 @@ def return_all( result_type=f"TUPLE({','.join(f'ARRAY({t})' for t in ALL_SCALAR_TYPES)})", ) def return_all_arrays( - bool, - i8, - i16, - i32, - i64, - u8, - u16, - u32, - u64, - f32, - f64, - date, - timestamp, - varchar, - json, + bool, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f32, + f64, + date, + timestamp, + varchar, + json, ): return ( bool, @@ -274,21 +273,21 @@ def return_all_arrays( result_type=f"TUPLE({','.join(f'{t}' for t in ALL_SCALAR_TYPES)})", ) def return_all_non_nullable( - bool, - i8, - i16, - i32, - i64, - u8, - u16, - u32, - u64, - f32, - f64, - date, - timestamp, - varchar, - json, + bool, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f32, + f64, + date, + timestamp, + varchar, + json, ): return ( bool, @@ -321,6 +320,11 @@ def wait_concurrent(x): return x +@udf(input_types=["VARCHAR"], result_type="VARCHAR") +def ping(s: str) -> str: + return s + + if __name__ == "__main__": udf_server = UDFServer("0.0.0.0:8815") udf_server.add_function(add_signed) @@ -347,4 +351,7 @@ def wait_concurrent(x): udf_server.add_function(wait) udf_server.add_function(wait_concurrent) udf_server.add_function(url_len) + + # Built-in function + udf_server.add_function(ping) udf_server.serve()