From c4a4e5f3375dc144235f90eb2ae00368495936af Mon Sep 17 00:00:00 2001 From: Chloe Kim Date: Fri, 5 May 2023 13:28:27 +0800 Subject: [PATCH] feat: `JSON_VALUE()` & `JSON_QUERY()` sql support (#1526) * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * feat: JSON_VALUE() and JSON_QUERY() sql function support Signed-off-by: Chloe Kim * added test cases Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * jsonpath implementation Signed-off-by: Chloe Kim * added jsonpath readme Signed-off-by: Chloe Kim --------- Signed-off-by: Chloe Kim --- Cargo.lock | 42 +- dozer-sql/Cargo.toml | 4 + dozer-sql/src/jsonpath/README.md | 3 + dozer-sql/src/jsonpath/mod.rs | 222 ++++++ .../jsonpath/parser/grammar/json_path.pest | 53 ++ dozer-sql/src/jsonpath/parser/macros.rs | 83 +++ dozer-sql/src/jsonpath/parser/mod.rs | 7 + dozer-sql/src/jsonpath/parser/model.rs | 177 +++++ dozer-sql/src/jsonpath/parser/parser.rs | 196 ++++++ dozer-sql/src/jsonpath/path/index.rs | 332 +++++++++ dozer-sql/src/jsonpath/path/json.rs | 162 +++++ dozer-sql/src/jsonpath/path/mod.rs | 76 ++ dozer-sql/src/jsonpath/path/top.rs | 314 +++++++++ dozer-sql/src/lib.rs | 5 + dozer-sql/src/pipeline/expression/builder.rs | 34 +- dozer-sql/src/pipeline/expression/cast.rs | 25 +- .../src/pipeline/expression/execution.rs | 24 + .../src/pipeline/expression/json_functions.rs | 136 ++++ dozer-sql/src/pipeline/expression/mod.rs | 1 + .../expression/tests/json_functions.rs | 647 ++++++++++++++++++ .../src/pipeline/expression/tests/mod.rs | 2 + dozer-types/src/json_types.rs | 206 +++++- dozer-types/src/types/field.rs | 62 +- dozer-types/src/types/tests.rs | 22 +- 24 files changed, 2801 insertions(+), 34 deletions(-) create mode 100644 dozer-sql/src/jsonpath/README.md create mode 100644 dozer-sql/src/jsonpath/mod.rs create mode 100644 dozer-sql/src/jsonpath/parser/grammar/json_path.pest create mode 100644 dozer-sql/src/jsonpath/parser/macros.rs create mode 100644 dozer-sql/src/jsonpath/parser/mod.rs create mode 100644 dozer-sql/src/jsonpath/parser/model.rs create mode 100644 dozer-sql/src/jsonpath/parser/parser.rs create mode 100644 dozer-sql/src/jsonpath/path/index.rs create mode 100644 dozer-sql/src/jsonpath/path/json.rs create mode 100644 dozer-sql/src/jsonpath/path/mod.rs create mode 100644 dozer-sql/src/jsonpath/path/top.rs create mode 100644 dozer-sql/src/pipeline/expression/json_functions.rs create mode 100644 dozer-sql/src/pipeline/expression/tests/json_functions.rs diff --git a/Cargo.lock b/Cargo.lock index e90c1b6c34..88fd9ed1bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -583,7 +583,7 @@ dependencies = [ "arrow-schema", "arrow-select", "regex", - "regex-syntax", + "regex-syntax 0.6.28", ] [[package]] @@ -1949,7 +1949,7 @@ dependencies = [ "datafusion-physical-expr", "hashbrown 0.13.2", "log", - "regex-syntax", + "regex-syntax 0.6.28", ] [[package]] @@ -2319,11 +2319,15 @@ dependencies = [ "dyn-clone", "enum_dispatch", "hashbrown 0.13.2", + "jsonpath-rust", "like", "linked-hash-map", "multimap", "num-traits", + "pest", + "pest_derive", "proptest", + "regex", "sqlparser 0.32.0", "tempdir", "uuid", @@ -3472,6 +3476,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonpath-rust" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7ea2fa3ba7d1404aa6b094aceec1d49106ec0110b40c40b76cedae148837a3b" +dependencies = [ + "pest", + "pest_derive", + "regex", + "serde_json", +] + [[package]] name = "jsonrpc-core" version = "18.0.0" @@ -3775,7 +3791,7 @@ dependencies = [ "fnv", "proc-macro2", "quote", - "regex-syntax", + "regex-syntax 0.6.28", "syn", ] @@ -4898,7 +4914,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.6.28", "rusty-fork", "tempfile", "unarray", @@ -5245,13 +5261,13 @@ checksum = "f4ed1d73fb92eba9b841ba2aef69533a060ccc0d3ec71c90aeda5996d4afb7a9" [[package]] name = "regex" -version = "1.7.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] [[package]] @@ -5260,7 +5276,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.28", ] [[package]] @@ -5269,6 +5285,12 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "remove_dir_all" version = "0.5.3" diff --git a/dozer-sql/Cargo.toml b/dozer-sql/Cargo.toml index cdf6040a6b..69b808f178 100644 --- a/dozer-sql/Cargo.toml +++ b/dozer-sql/Cargo.toml @@ -22,6 +22,10 @@ ahash = "0.8.3" bloom = "0.3.2" enum_dispatch = "0.3.11" linked-hash-map = "0.5.6" +pest_derive = "2.5.6" +pest = "2.5.6" +jsonpath-rust = "0.3.0" +regex = "1.8.1" [dev-dependencies] tempdir = "0.3.7" diff --git a/dozer-sql/src/jsonpath/README.md b/dozer-sql/src/jsonpath/README.md new file mode 100644 index 0000000000..63d6c29056 --- /dev/null +++ b/dozer-sql/src/jsonpath/README.md @@ -0,0 +1,3 @@ +## Disclaimer + +This `jsonpath` folder has been forked from [jsonpath-rust](https://github.com/besok/jsonpath-rust) to integrate with Dozer native `JsonValue`. diff --git a/dozer-sql/src/jsonpath/mod.rs b/dozer-sql/src/jsonpath/mod.rs new file mode 100644 index 0000000000..f6bb58226f --- /dev/null +++ b/dozer-sql/src/jsonpath/mod.rs @@ -0,0 +1,222 @@ +#![allow(clippy::vec_init_then_push)] + +use crate::jsonpath::parser::model::JsonPath; +use crate::jsonpath::path::{json_path_instance, PathInstance}; +use crate::jsonpath::JsonPathValue::{NewValue, NoValue, Slice}; +use dozer_types::json_types::JsonValue; +use std::convert::TryInto; +use std::fmt::Debug; +use std::str::FromStr; + +pub mod parser; +pub mod path; + +pub trait JsonPathQuery { + fn path(self, query: &str) -> Result; +} + +pub struct JsonPathInst { + inner: JsonPath, +} + +impl FromStr for JsonPathInst { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(JsonPathInst { + inner: s.try_into()?, + }) + } +} + +impl JsonPathQuery for Box { + fn path(self, query: &str) -> Result { + let p = JsonPathInst::from_str(query)?; + Ok(JsonPathFinder::new(self, Box::new(p)).find()) + } +} + +impl JsonPathQuery for JsonValue { + fn path(self, query: &str) -> Result { + let p = JsonPathInst::from_str(query)?; + Ok(JsonPathFinder::new(Box::new(self), Box::new(p)).find()) + } +} + +#[macro_export] +macro_rules! json_path_value { + (&$v:expr) =>{ + JsonPathValue::Slice(&$v) + }; + + ($(&$v:expr),+ $(,)?) =>{ + { + let mut res = Vec::new(); + $( + res.push(json_path_value!(&$v)); + )+ + res + } + }; + ($v:expr) =>{ + JsonPathValue::NewValue($v) + }; + +} + +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum JsonPathValue<'a, Data> { + /// The slice of the initial json data + Slice(&'a Data), + /// The new data that was generated from the input data (like length operator) + NewValue(Data), + /// The absent value that indicates the input data is not matched to the given json path (like the absent fields) + NoValue, +} + +impl<'a, Data: Clone + Debug + Default> JsonPathValue<'a, Data> { + pub fn to_data(self) -> Data { + match self { + Slice(r) => r.clone(), + NewValue(val) => val, + NoValue => Data::default(), + } + } +} + +impl<'a, Data> From<&'a Data> for JsonPathValue<'a, Data> { + fn from(data: &'a Data) -> Self { + Slice(data) + } +} + +impl<'a, Data> JsonPathValue<'a, Data> { + fn only_no_value(input: &Vec>) -> bool { + !input.is_empty() && input.iter().filter(|v| v.has_value()).count() == 0 + } + + fn map_vec(data: Vec<&'a Data>) -> Vec> { + data.into_iter().map(|v| v.into()).collect() + } + + fn map_slice(self, mapper: F) -> Vec> + where + F: FnOnce(&'a Data) -> Vec<&'a Data>, + { + match self { + Slice(r) => mapper(r).into_iter().map(Slice).collect(), + NewValue(_) => vec![], + no_v => vec![no_v], + } + } + + fn flat_map_slice(self, mapper: F) -> Vec> + where + F: FnOnce(&'a Data) -> Vec>, + { + match self { + Slice(r) => mapper(r), + _ => vec![NoValue], + } + } + + pub fn has_value(&self) -> bool { + !matches!(self, NoValue) + } + + pub fn into_data(input: Vec>) -> Vec<&'a Data> { + input + .into_iter() + .filter_map(|v| match v { + Slice(el) => Some(el), + _ => None, + }) + .collect() + } + + /// moves a pointer (from slice) out or provides a default value when the value was generated + pub fn slice_or(self, default: &'a Data) -> &'a Data { + match self { + Slice(r) => r, + NewValue(_) | NoValue => default, + } + } +} + +/// The base structure stitching the json instance and jsonpath instance +pub struct JsonPathFinder { + json: Box, + path: Box, +} + +impl JsonPathFinder { + /// creates a new instance of [JsonPathFinder] + pub fn new(json: Box, path: Box) -> Self { + JsonPathFinder { json, path } + } + + /// updates a path with a new one + pub fn set_path(&mut self, path: Box) { + self.path = path + } + /// updates a json with a new one + pub fn set_json(&mut self, json: Box) { + self.json = json + } + /// updates a json from string and therefore can be some parsing errors + pub fn set_json_str(&mut self, json: &str) -> Result<(), String> { + self.json = Box::from(JsonValue::from_str(json).map_err(|e| e.to_string())?); + Ok(()) + } + /// updates a path from string and therefore can be some parsing errors + pub fn set_path_str(&mut self, path: &str) -> Result<(), String> { + self.path = Box::new(JsonPathInst::from_str(path)?); + Ok(()) + } + + /// create a new instance from string and therefore can be some parsing errors + pub fn from_str(json: &str, path: &str) -> Result { + let json = JsonValue::from_str(json).map_err(|e| e.to_string())?; + let path = Box::new(JsonPathInst::from_str(path)?); + Ok(JsonPathFinder::new(Box::from(json), path)) + } + + /// creates an instance to find a json slice from the json + pub fn instance(&self) -> PathInstance { + json_path_instance(&self.path.inner, &self.json) + } + /// finds a slice of data in the set json. + /// The result is a vector of references to the incoming structure. + pub fn find_slice(&self) -> Vec> { + let res = self.instance().find((&(*self.json)).into()); + let has_v: Vec> = + res.into_iter().filter(|v| v.has_value()).collect(); + + if has_v.is_empty() { + vec![NoValue] + } else { + has_v + } + } + + /// finds a slice of data and wrap it with Value::Array by cloning the data. + /// Returns either an array of elements or Json::Null if the match is incorrect. + pub fn find(&self) -> JsonValue { + let slice = self.find_slice(); + if !slice.is_empty() { + if JsonPathValue::only_no_value(&slice) { + JsonValue::Null + } else { + JsonValue::Array( + self.find_slice() + .into_iter() + .filter(|v| v.has_value()) + .map(|v| v.to_data()) + .collect(), + ) + } + } else { + JsonValue::Array(vec![]) + } + } +} diff --git a/dozer-sql/src/jsonpath/parser/grammar/json_path.pest b/dozer-sql/src/jsonpath/parser/grammar/json_path.pest new file mode 100644 index 0000000000..a0ed9594b6 --- /dev/null +++ b/dozer-sql/src/jsonpath/parser/grammar/json_path.pest @@ -0,0 +1,53 @@ +WHITESPACE = _{ " " | "\t" | "\r\n" | "\n"} + +boolean = {"true" | "false"} +null = {"null"} + +min = _{"-"} +col = _{":"} +dot = _{ "." } +word = _{ ('a'..'z' | 'A'..'Z')+ } +specs = _{ "_" | "-" | "/" | "\\" | "#" } +number = @{"-"? ~ ("0" | ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*) ~ ("." ~ ASCII_DIGIT+)? ~ (^"e" ~ ("+" | "-")? ~ ASCII_DIGIT+)?} + +string_qt = ${ "\'" ~ inner ~ "\'" } +inner = @{ char* } +char = _{ + !("\"" | "\\" | "\'") ~ ANY + | "\\" ~ ("\"" | "\'" | "\\" | "/" | "b" | "f" | "n" | "r" | "t") + | "\\" ~ ("u" ~ ASCII_HEX_DIGIT{4}) +} +root = {"$"} +sign = { "==" | "!=" | "~=" | ">=" | ">" | "<=" | "<" | "in" | "nin" | "size" | "noneOf" | "anyOf" | "subsetOf"} +key_lim = {!"length()" ~ (word | ASCII_DIGIT | specs)+} +key_unlim = {"[" ~ string_qt ~ "]"} +key = ${key_lim | key_unlim} + +descent = {dot ~ dot ~ key} +descent_w = {dot ~ dot ~ "*"} // refactor afterwards +wildcard = {dot? ~ "[" ~"*"~"]" | dot ~ "*"} +current = {"@" ~ chain?} +field = ${dot? ~ key_unlim | dot ~ key_lim } +function = { dot ~ "length" ~ "(" ~ ")"} +unsigned = {("0" | ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*)} +signed = {min? ~ unsigned} +start_slice = {signed} +end_slice = {signed} +step_slice = {col ~ unsigned} +slice = {start_slice? ~ col ~ end_slice? ~ step_slice? } + +unit_keys = { string_qt ~ ("," ~ string_qt)+ } +unit_indexes = { number ~ ("," ~ number)+ } +filter = {"?"~ "(" ~ logic ~ ")"} + +logic = {logic_and ~ ("||" ~ logic_and)*} +logic_and = {logic_atom ~ ("&&" ~ logic_atom)*} +logic_atom = {atom ~ (sign ~ atom)? | "(" ~ logic ~ ")"} + +atom = {chain | string_qt | number | boolean | null} + +index = {dot? ~ "["~ (unit_keys | unit_indexes | slice | unsigned |filter) ~ "]" } + +chain = {(root | descent | descent_w | wildcard | current | field | index | function)+} + +path = {SOI ~ chain ~ EOI } \ No newline at end of file diff --git a/dozer-sql/src/jsonpath/parser/macros.rs b/dozer-sql/src/jsonpath/parser/macros.rs new file mode 100644 index 0000000000..e8847005aa --- /dev/null +++ b/dozer-sql/src/jsonpath/parser/macros.rs @@ -0,0 +1,83 @@ +#[macro_export] +macro_rules! filter { + () => {FilterExpression::Atom(op!,FilterSign::new(""),op!())}; + ( $left:expr, $s:literal, $right:expr) => { + FilterExpression::Atom($left,FilterSign::new($s),$right) + }; + ( $left:expr,||, $right:expr) => {FilterExpression::Or(Box::new($left),Box::new($right)) }; + ( $left:expr,&&, $right:expr) => {FilterExpression::And(Box::new($left),Box::new($right)) }; +} +#[macro_export] +macro_rules! op { + ( ) => { + Operand::Dynamic(Box::new(JsonPath::Empty)) + }; + ( $s:literal) => { + Operand::Static(json!($s)) + }; + ( s $s:expr) => { + Operand::Static(json!($s)) + }; + ( $s:expr) => { + Operand::Dynamic(Box::new($s)) + }; +} + +#[macro_export] +macro_rules! idx { + ( $s:literal) => {JsonPathIndex::Single(json!($s))}; + ( idx $($ss:literal),+) => {{ + let mut ss_vec = Vec::new(); + $( ss_vec.push(json!($ss)) ; )+ + JsonPathIndex::UnionIndex(ss_vec) + }}; + ( $($ss:literal),+) => {{ + let mut ss_vec = Vec::new(); + $( ss_vec.push($ss.to_string()) ; )+ + JsonPathIndex::UnionKeys(ss_vec) + }}; + ( $s:literal) => {JsonPathIndex::Single(json!($s))}; + ( ? $s:expr) => {JsonPathIndex::Filter($s)}; + ( [$l:literal;$m:literal;$r:literal]) => {JsonPathIndex::Slice($l,$m,$r)}; + ( [$l:literal;$m:literal;]) => {JsonPathIndex::Slice($l,$m,1)}; + ( [$l:literal;;$m:literal]) => {JsonPathIndex::Slice($l,0,$m)}; + ( [;$l:literal;$m:literal]) => {JsonPathIndex::Slice(0,$l,$m)}; + ( [;;$m:literal]) => {JsonPathIndex::Slice(0,0,$m)}; + ( [;$m:literal;]) => {JsonPathIndex::Slice(0,$m,1)}; + ( [$m:literal;;]) => {JsonPathIndex::Slice($m,0,1)}; + ( [;;]) => {JsonPathIndex::Slice(0,0,1)}; +} + +#[macro_export] +macro_rules! chain { + ($($ss:expr),+) => {{ + let mut ss_vec = Vec::new(); + $( ss_vec.push($ss) ; )+ + JsonPath::Chain(ss_vec) + }}; +} + +#[macro_export] +macro_rules! path { + ( ) => {JsonPath::Empty}; + (*) => {JsonPath::Wildcard}; + ($) => {JsonPath::Root}; + (@) => {JsonPath::Current(Box::new(JsonPath::Empty))}; + (@$e:expr) => {JsonPath::Current(Box::new($e))}; + (@,$($ss:expr),+) => {{ + let mut ss_vec = Vec::new(); + $( ss_vec.push($ss) ; )+ + let chain = JsonPath::Chain(ss_vec); + JsonPath::Current(Box::new(chain)) + }}; + (..$e:literal) => {JsonPath::Descent($e.to_string())}; + (..*) => {JsonPath::DescentW}; + ($e:literal) => {JsonPath::Field($e.to_string())}; + ($e:expr) => {JsonPath::Index($e)}; +} +#[macro_export] +macro_rules! function { + (length) => { + JsonPath::Fn(Function::Length) + }; +} diff --git a/dozer-sql/src/jsonpath/parser/mod.rs b/dozer-sql/src/jsonpath/parser/mod.rs new file mode 100644 index 0000000000..9077ad6c64 --- /dev/null +++ b/dozer-sql/src/jsonpath/parser/mod.rs @@ -0,0 +1,7 @@ +//! The parser for the jsonpath. +//! The module grammar denotes the structure of the parsing grammar + +mod macros; +pub mod model; +#[allow(clippy::module_inception)] +pub mod parser; diff --git a/dozer-sql/src/jsonpath/parser/model.rs b/dozer-sql/src/jsonpath/parser/model.rs new file mode 100644 index 0000000000..56e5b1aff7 --- /dev/null +++ b/dozer-sql/src/jsonpath/parser/model.rs @@ -0,0 +1,177 @@ +use crate::jsonpath::parser::parser::parse_json_path; +use dozer_types::json_types::JsonValue; +use std::convert::TryFrom; + +/// The basic structures for parsing json paths. +/// The common logic of the structures pursues to correspond the internal parsing structure. +#[derive(Debug, Clone)] +pub enum JsonPath { + /// The $ operator + Root, + /// Field represents key + Field(String), + /// The whole chain of the path. + Chain(Vec), + /// The .. operator + Descent(String), + /// The ..* operator + DescentW, + /// The indexes for array + Index(JsonPathIndex), + /// The @ operator + Current(Box), + /// The * operator + Wildcard, + /// The item uses to define the unresolved state + Empty, + /// Functions that can calculate some expressions + Fn(Function), +} + +impl TryFrom<&str> for JsonPath { + type Error = String; + + fn try_from(value: &str) -> Result { + parse_json_path(value).map_err(|e| e.to_string()) + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum Function { + /// length() + Length, +} +#[derive(Debug, Clone)] +pub enum JsonPathIndex { + /// A single element in array + Single(JsonValue), + /// Union represents a several indexes + UnionIndex(Vec), + /// Union represents a several keys + UnionKeys(Vec), + /// DEfault slice where the items are start/end/step respectively + Slice(i32, i32, usize), + /// Filter ?() + Filter(FilterExpression), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FilterExpression { + /// a single expression like a > 2 + Atom(Operand, FilterSign, Operand), + /// and with && + And(Box, Box), + /// or with || + Or(Box, Box), +} + +impl FilterExpression { + pub fn exists(op: Operand) -> Self { + FilterExpression::Atom( + op, + FilterSign::Exists, + Operand::Dynamic(Box::new(JsonPath::Empty)), + ) + } +} + +/// Operand for filtering expressions +#[derive(Debug, Clone)] +pub enum Operand { + Static(JsonValue), + Dynamic(Box), +} + +#[allow(dead_code)] +impl Operand { + pub fn val(v: JsonValue) -> Self { + Operand::Static(v) + } +} + +/// The operators for filtering functions +#[derive(Debug, Clone, PartialEq)] +pub enum FilterSign { + Equal, + Unequal, + Less, + Greater, + LeOrEq, + GrOrEq, + Regex, + In, + Nin, + Size, + NoneOf, + AnyOf, + SubSetOf, + Exists, +} + +impl FilterSign { + pub fn new(key: &str) -> Self { + match key { + "==" => FilterSign::Equal, + "!=" => FilterSign::Unequal, + "<" => FilterSign::Less, + ">" => FilterSign::Greater, + "<=" => FilterSign::LeOrEq, + ">=" => FilterSign::GrOrEq, + "~=" => FilterSign::Regex, + "in" => FilterSign::In, + "nin" => FilterSign::Nin, + "size" => FilterSign::Size, + "noneOf" => FilterSign::NoneOf, + "anyOf" => FilterSign::AnyOf, + "subsetOf" => FilterSign::SubSetOf, + _ => FilterSign::Exists, + } + } +} + +impl PartialEq for JsonPath { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (JsonPath::Root, JsonPath::Root) => true, + (JsonPath::Descent(k1), JsonPath::Descent(k2)) => k1 == k2, + (JsonPath::DescentW, JsonPath::DescentW) => true, + (JsonPath::Field(k1), JsonPath::Field(k2)) => k1 == k2, + (JsonPath::Wildcard, JsonPath::Wildcard) => true, + (JsonPath::Empty, JsonPath::Empty) => true, + (JsonPath::Current(jp1), JsonPath::Current(jp2)) => jp1 == jp2, + (JsonPath::Chain(ch1), JsonPath::Chain(ch2)) => ch1 == ch2, + (JsonPath::Index(idx1), JsonPath::Index(idx2)) => idx1 == idx2, + (JsonPath::Fn(fn1), JsonPath::Fn(fn2)) => fn2 == fn1, + (_, _) => false, + } + } +} + +impl PartialEq for JsonPathIndex { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (JsonPathIndex::Slice(s1, e1, st1), JsonPathIndex::Slice(s2, e2, st2)) => { + s1 == s2 && e1 == e2 && st1 == st2 + } + (JsonPathIndex::Single(el1), JsonPathIndex::Single(el2)) => el1 == el2, + (JsonPathIndex::UnionIndex(elems1), JsonPathIndex::UnionIndex(elems2)) => { + elems1 == elems2 + } + (JsonPathIndex::UnionKeys(elems1), JsonPathIndex::UnionKeys(elems2)) => { + elems1 == elems2 + } + (JsonPathIndex::Filter(left), JsonPathIndex::Filter(right)) => left.eq(right), + (_, _) => false, + } + } +} + +impl PartialEq for Operand { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Operand::Static(v1), Operand::Static(v2)) => v1 == v2, + (Operand::Dynamic(jp1), Operand::Dynamic(jp2)) => jp1 == jp2, + (_, _) => false, + } + } +} diff --git a/dozer-sql/src/jsonpath/parser/parser.rs b/dozer-sql/src/jsonpath/parser/parser.rs new file mode 100644 index 0000000000..8dc79c9a82 --- /dev/null +++ b/dozer-sql/src/jsonpath/parser/parser.rs @@ -0,0 +1,196 @@ +use crate::jsonpath::parser::model::FilterExpression::{And, Or}; +use crate::jsonpath::parser::model::{ + FilterExpression, FilterSign, Function, JsonPath, JsonPathIndex, Operand, +}; +use dozer_types::json_types::JsonValue; +use pest::error::Error; +use pest::iterators::{Pair, Pairs}; +use pest::Parser; + +#[derive(Parser)] +#[grammar = "jsonpath/parser/grammar/json_path.pest"] +pub struct JsonPathParser; + +/// the parsing function. +/// Since the parsing can finish with error the result is [[Result]] +#[allow(clippy::result_large_err)] +pub fn parse_json_path(jp_str: &str) -> Result> { + Ok(parse_internal( + JsonPathParser::parse(Rule::path, jp_str)?.next().unwrap(), + )) +} + +/// Internal function takes care of the logic by parsing the operators and unrolling the string into the final result. +fn parse_internal(rule: Pair) -> JsonPath { + match rule.as_rule() { + Rule::path => rule + .into_inner() + .next() + .map(parse_internal) + .unwrap_or(JsonPath::Empty), + Rule::current => JsonPath::Current(Box::new( + rule.into_inner() + .next() + .map(parse_internal) + .unwrap_or(JsonPath::Empty), + )), + Rule::chain => JsonPath::Chain(rule.into_inner().map(parse_internal).collect()), + Rule::root => JsonPath::Root, + Rule::wildcard => JsonPath::Wildcard, + Rule::descent => parse_key(down(rule)) + .map(JsonPath::Descent) + .unwrap_or(JsonPath::Empty), + Rule::descent_w => JsonPath::DescentW, + Rule::function => JsonPath::Fn(Function::Length), + Rule::field => parse_key(down(rule)) + .map(JsonPath::Field) + .unwrap_or(JsonPath::Empty), + Rule::index => JsonPath::Index(parse_index(rule)), + _ => JsonPath::Empty, + } +} + +/// parsing the rule 'key' with the structures either .key or .]'key'[ +fn parse_key(rule: Pair) -> Option { + match rule.as_rule() { + Rule::key | Rule::key_unlim | Rule::string_qt => parse_key(down(rule)), + Rule::key_lim | Rule::inner => Some(String::from(rule.as_str())), + _ => None, + } +} + +fn parse_slice(mut pairs: Pairs) -> JsonPathIndex { + let mut start = 0; + let mut end = 0; + let mut step = 1; + while pairs.peek().is_some() { + let in_pair = pairs.next().unwrap(); + match in_pair.as_rule() { + Rule::start_slice => start = in_pair.as_str().parse::().unwrap_or(start), + Rule::end_slice => end = in_pair.as_str().parse::().unwrap_or(end), + Rule::step_slice => step = down(in_pair).as_str().parse::().unwrap_or(step), + _ => (), + } + } + JsonPathIndex::Slice(start, end, step) +} + +fn parse_unit_keys(mut pairs: Pairs) -> JsonPathIndex { + let mut keys = vec![]; + + while pairs.peek().is_some() { + keys.push(String::from(down(pairs.next().unwrap()).as_str())); + } + JsonPathIndex::UnionKeys(keys) +} + +fn number_to_value(number: &str) -> JsonValue { + number.parse::().ok().map(JsonValue::from).unwrap() +} + +fn parse_unit_indexes(mut pairs: Pairs) -> JsonPathIndex { + let mut keys = vec![]; + + while pairs.peek().is_some() { + keys.push(number_to_value(pairs.next().unwrap().as_str())); + } + JsonPathIndex::UnionIndex(keys) +} + +fn parse_chain_in_operand(rule: Pair) -> Operand { + match parse_internal(rule) { + JsonPath::Chain(elems) => { + if elems.len() == 1 { + match elems.first() { + Some(JsonPath::Index(JsonPathIndex::UnionKeys(keys))) => { + Operand::val(JsonValue::from(keys.clone())) + } + Some(JsonPath::Index(JsonPathIndex::UnionIndex(keys))) => { + Operand::val(JsonValue::from(keys.clone())) + } + Some(JsonPath::Field(f)) => { + Operand::val(JsonValue::Array(vec![JsonValue::from(f.clone())])) + } + _ => Operand::Dynamic(Box::new(JsonPath::Chain(elems))), + } + } else { + Operand::Dynamic(Box::new(JsonPath::Chain(elems))) + } + } + jp => Operand::Dynamic(Box::new(jp)), + } +} + +fn parse_filter_index(pair: Pair) -> JsonPathIndex { + JsonPathIndex::Filter(parse_logic(pair.into_inner())) +} + +fn parse_logic(mut pairs: Pairs) -> FilterExpression { + let mut expr: Option = None; + while pairs.peek().is_some() { + let next_expr = parse_logic_and(pairs.next().unwrap().into_inner()); + match expr { + None => expr = Some(next_expr), + Some(e) => expr = Some(Or(Box::new(e), Box::new(next_expr))), + } + } + expr.unwrap() +} + +fn parse_logic_and(mut pairs: Pairs) -> FilterExpression { + let mut expr: Option = None; + + while pairs.peek().is_some() { + let next_expr = parse_logic_atom(pairs.next().unwrap().into_inner()); + match expr { + None => expr = Some(next_expr), + Some(e) => expr = Some(And(Box::new(e), Box::new(next_expr))), + } + } + expr.unwrap() +} + +fn parse_logic_atom(mut pairs: Pairs) -> FilterExpression { + match pairs.peek().map(|x| x.as_rule()) { + Some(Rule::logic) => parse_logic(pairs.next().unwrap().into_inner()), + Some(Rule::atom) => { + let left: Operand = parse_atom(pairs.next().unwrap()); + if pairs.peek().is_none() { + FilterExpression::exists(left) + } else { + let sign: FilterSign = FilterSign::new(pairs.next().unwrap().as_str()); + let right: Operand = parse_atom(pairs.next().unwrap()); + FilterExpression::Atom(left, sign, right) + } + } + Some(x) => panic!("unexpected => {:?}", x), + None => panic!("unexpected none"), + } +} + +fn parse_atom(rule: Pair) -> Operand { + let atom = down(rule.clone()); + match atom.as_rule() { + Rule::number => Operand::Static(number_to_value(rule.as_str())), + Rule::string_qt => Operand::Static(JsonValue::from(down(atom).as_str())), + Rule::chain => parse_chain_in_operand(down(rule)), + Rule::boolean => Operand::Static(rule.as_str().parse().unwrap()), + _ => Operand::Static(JsonValue::Null), + } +} + +fn parse_index(rule: Pair) -> JsonPathIndex { + let next = down(rule); + match next.as_rule() { + Rule::unsigned => JsonPathIndex::Single(number_to_value(next.as_str())), + Rule::slice => parse_slice(next.into_inner()), + Rule::unit_indexes => parse_unit_indexes(next.into_inner()), + Rule::unit_keys => parse_unit_keys(next.into_inner()), + Rule::filter => parse_filter_index(down(next)), + _ => JsonPathIndex::Single(number_to_value(next.as_str())), + } +} + +fn down(rule: Pair) -> Pair { + rule.into_inner().next().unwrap() +} diff --git a/dozer-sql/src/jsonpath/path/index.rs b/dozer-sql/src/jsonpath/path/index.rs new file mode 100644 index 0000000000..859466c32d --- /dev/null +++ b/dozer-sql/src/jsonpath/path/index.rs @@ -0,0 +1,332 @@ +use crate::jsonpath::parser::model::{FilterExpression, FilterSign, JsonPath}; +use crate::jsonpath::path::json::{any_of, eq, inside, less, regex, size, sub_set_of}; +use crate::jsonpath::path::top::ObjectField; +use crate::jsonpath::path::{json_path_instance, process_operand, Path, PathInstance}; +use crate::jsonpath::JsonPathValue; +use crate::jsonpath::JsonPathValue::{NoValue, Slice}; +use dozer_types::json_types::JsonValue; +use dozer_types::json_types::JsonValue::Array; + +/// process the slice like [start:end:step] +#[derive(Debug)] +pub(crate) struct ArraySlice { + start_index: i32, + end_index: i32, + step: usize, +} + +impl ArraySlice { + pub(crate) fn new(start_index: i32, end_index: i32, step: usize) -> ArraySlice { + ArraySlice { + start_index, + end_index, + step, + } + } + + fn end(&self, len: i32) -> Option { + if self.end_index >= 0 { + if self.end_index > len { + None + } else { + Some(self.end_index as usize) + } + } else if self.end_index < -len { + None + } else { + Some((len - (-self.end_index)) as usize) + } + } + + fn start(&self, len: i32) -> Option { + if self.start_index >= 0 { + if self.start_index > len { + None + } else { + Some(self.start_index as usize) + } + } else if self.start_index < -len { + None + } else { + Some((len - -self.start_index) as usize) + } + } + + fn process<'a, T>(&self, elements: &'a [T]) -> Vec<&'a T> { + let len = elements.len() as i32; + let mut filtered_elems: Vec<&T> = vec![]; + match (self.start(len), self.end(len)) { + (Some(start_idx), Some(end_idx)) => { + let end_idx = if end_idx == 0 { + elements.len() + } else { + end_idx + }; + for idx in (start_idx..end_idx).step_by(self.step) { + if let Some(v) = elements.get(idx) { + filtered_elems.push(v) + } + } + filtered_elems + } + _ => filtered_elems, + } + } +} + +impl<'a> Path<'a> for ArraySlice { + type Data = JsonValue; + + fn find(&self, input: JsonPathValue<'a, Self::Data>) -> Vec> { + input.flat_map_slice(|data| { + data.as_array() + .map(|elems| self.process(elems)) + .and_then(|v| { + if v.is_empty() { + None + } else { + Some(JsonPathValue::map_vec(v)) + } + }) + .unwrap_or_else(|| vec![NoValue]) + }) + } +} + +/// process the simple index like [index] +pub(crate) struct ArrayIndex { + index: usize, +} + +impl ArrayIndex { + pub(crate) fn new(index: usize) -> Self { + ArrayIndex { index } + } +} + +impl<'a> Path<'a> for ArrayIndex { + type Data = JsonValue; + + fn find(&self, input: JsonPathValue<'a, Self::Data>) -> Vec> { + input.flat_map_slice(|data| { + data.as_array() + .and_then(|elems| elems.get(self.index)) + .map(|e| vec![e.into()]) + .unwrap_or_else(|| vec![NoValue]) + }) + } +} + +/// process @ element +pub(crate) struct Current<'a> { + tail: Option>, +} + +impl<'a> Current<'a> { + pub(crate) fn from(jp: &'a JsonPath, root: &'a JsonValue) -> Self { + match jp { + JsonPath::Empty => Current::none(), + tail => Current::new(json_path_instance(tail, root)), + } + } + pub(crate) fn new(tail: PathInstance<'a>) -> Self { + Current { tail: Some(tail) } + } + pub(crate) fn none() -> Self { + Current { tail: None } + } +} + +impl<'a> Path<'a> for Current<'a> { + type Data = JsonValue; + + fn find(&self, input: JsonPathValue<'a, Self::Data>) -> Vec> { + self.tail + .as_ref() + .map(|p| p.find(input.clone())) + .unwrap_or_else(|| vec![input]) + } +} + +/// the list of indexes like [1,2,3] +pub(crate) struct UnionIndex<'a> { + indexes: Vec>, +} + +impl<'a> UnionIndex<'a> { + pub fn from_indexes(elems: &'a [JsonValue]) -> Self { + let mut indexes: Vec> = vec![]; + + for idx in elems.iter() { + indexes.push(Box::new(ArrayIndex::new(idx.as_u64().unwrap() as usize))) + } + + UnionIndex::new(indexes) + } + pub fn from_keys(elems: &'a [String]) -> Self { + let mut indexes: Vec> = vec![]; + + for key in elems.iter() { + indexes.push(Box::new(ObjectField::new(key))) + } + + UnionIndex::new(indexes) + } + + pub fn new(indexes: Vec>) -> Self { + UnionIndex { indexes } + } +} + +impl<'a> Path<'a> for UnionIndex<'a> { + type Data = JsonValue; + + fn find(&self, input: JsonPathValue<'a, Self::Data>) -> Vec> { + self.indexes + .iter() + .flat_map(|e| e.find(input.clone())) + .collect() + } +} + +/// process filter element like [?(op sign op)] +pub enum FilterPath<'a> { + Filter { + left: PathInstance<'a>, + right: PathInstance<'a>, + op: &'a FilterSign, + }, + Or { + left: PathInstance<'a>, + right: PathInstance<'a>, + }, + And { + left: PathInstance<'a>, + right: PathInstance<'a>, + }, +} + +impl<'a> FilterPath<'a> { + pub(crate) fn new(expr: &'a FilterExpression, root: &'a JsonValue) -> Self { + match expr { + FilterExpression::Atom(left, op, right) => FilterPath::Filter { + left: process_operand(left, root), + right: process_operand(right, root), + op, + }, + FilterExpression::And(l, r) => FilterPath::And { + left: Box::new(FilterPath::new(l, root)), + right: Box::new(FilterPath::new(r, root)), + }, + FilterExpression::Or(l, r) => FilterPath::Or { + left: Box::new(FilterPath::new(l, root)), + right: Box::new(FilterPath::new(r, root)), + }, + } + } + fn compound( + one: &'a FilterSign, + two: &'a FilterSign, + left: Vec>, + right: Vec>, + ) -> bool { + FilterPath::process_atom(one, left.clone(), right.clone()) + || FilterPath::process_atom(two, left, right) + } + fn process_atom( + op: &'a FilterSign, + left: Vec>, + right: Vec>, + ) -> bool { + match op { + FilterSign::Equal => eq( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + FilterSign::Unequal => !FilterPath::process_atom(&FilterSign::Equal, left, right), + FilterSign::Less => less( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + FilterSign::LeOrEq => { + FilterPath::compound(&FilterSign::Less, &FilterSign::Equal, left, right) + } + FilterSign::Greater => !FilterPath::process_atom(&FilterSign::LeOrEq, left, right), + FilterSign::GrOrEq => !FilterPath::process_atom(&FilterSign::Less, left, right), + FilterSign::Regex => regex( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + FilterSign::In => inside( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + FilterSign::Nin => !FilterPath::process_atom(&FilterSign::In, left, right), + FilterSign::NoneOf => !FilterPath::process_atom(&FilterSign::AnyOf, left, right), + FilterSign::AnyOf => any_of( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + FilterSign::SubSetOf => sub_set_of( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + FilterSign::Exists => !JsonPathValue::into_data(left).is_empty(), + FilterSign::Size => size( + JsonPathValue::into_data(left), + JsonPathValue::into_data(right), + ), + } + } + + fn process(&self, curr_el: &'a JsonValue) -> bool { + match self { + FilterPath::Filter { left, right, op } => { + FilterPath::process_atom(op, left.find(Slice(curr_el)), right.find(Slice(curr_el))) + } + FilterPath::Or { left, right } => { + if !JsonPathValue::into_data(left.find(Slice(curr_el))).is_empty() { + true + } else { + !JsonPathValue::into_data(right.find(Slice(curr_el))).is_empty() + } + } + FilterPath::And { left, right } => { + if JsonPathValue::into_data(left.find(Slice(curr_el))).is_empty() { + false + } else { + !JsonPathValue::into_data(right.find(Slice(curr_el))).is_empty() + } + } + } + } +} + +impl<'a> Path<'a> for FilterPath<'a> { + type Data = JsonValue; + + fn find(&self, input: JsonPathValue<'a, Self::Data>) -> Vec> { + input.flat_map_slice(|data| { + let mut res = vec![]; + match data { + Array(elems) => { + for el in elems.iter() { + if self.process(el) { + res.push(Slice(el)) + } + } + } + el => { + if self.process(el) { + res.push(Slice(el)) + } + } + } + if res.is_empty() { + vec![NoValue] + } else { + res + } + }) + } +} diff --git a/dozer-sql/src/jsonpath/path/json.rs b/dozer-sql/src/jsonpath/path/json.rs new file mode 100644 index 0000000000..0a8686597c --- /dev/null +++ b/dozer-sql/src/jsonpath/path/json.rs @@ -0,0 +1,162 @@ +use dozer_types::json_types::JsonValue; +use regex::Regex; + +/// compare sizes of json elements +/// The method expects to get a number on the right side and array or string or object on the left +/// where the number of characters, elements or fields will be compared respectively. +pub fn size(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if let Some(JsonValue::Number(n)) = right.get(0) { + for el in left.iter() { + match el { + JsonValue::String(v) if v.len() == n.0 as usize => true, + JsonValue::Array(elems) if elems.len() == n.0 as usize => true, + JsonValue::Object(fields) if fields.len() == n.0 as usize => true, + _ => return false, + }; + } + return true; + } + false +} + +/// ensure the array on the left side is a subset of the array on the right side. +pub fn sub_set_of(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if left.is_empty() { + return true; + } + if right.is_empty() { + return false; + } + + if let Some(elems) = left.first().and_then(|e| (*e).as_array()) { + if let Some(JsonValue::Array(right_elems)) = right.get(0) { + if right_elems.is_empty() { + return false; + } + + for el in elems { + let mut res = false; + + for r in right_elems.iter() { + if el.eq(r) { + res = true + } + } + if !res { + return false; + } + } + return true; + } + } + false +} + +/// ensure at least one element in the array on the left side belongs to the array on the right side. +pub fn any_of(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if left.is_empty() { + return true; + } + if right.is_empty() { + return false; + } + + if let Some(JsonValue::Array(elems)) = right.get(0) { + if elems.is_empty() { + return false; + } + + for el in left.iter() { + if let Some(left_elems) = el.as_array() { + for l in left_elems.iter() { + for r in elems.iter() { + if l.eq(r) { + return true; + } + } + } + } else { + for r in elems.iter() { + if el.eq(&r) { + return true; + } + } + } + } + } + + false +} + +/// ensure that the element on the left sides mathes the regex on the right side +pub fn regex(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if left.is_empty() || right.is_empty() { + return false; + } + + match right.get(0) { + Some(JsonValue::String(str)) => { + if let Ok(regex) = Regex::new(str) { + for el in left.iter() { + if let Some(v) = el.as_str() { + if regex.is_match(v) { + return true; + } + } + } + } + false + } + _ => false, + } +} + +/// ensure that the element on the left side belongs to the array on the right side. +pub fn inside(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if left.is_empty() { + return false; + } + + match right.get(0) { + Some(JsonValue::Array(elems)) => { + for el in left.iter() { + if elems.contains(el) { + return true; + } + } + false + } + Some(JsonValue::Object(elems)) => { + for el in left.iter() { + for r in elems.values() { + if el.eq(&r) { + return true; + } + } + } + false + } + _ => false, + } +} + +/// ensure the number on the left side is less the number on the right side +pub fn less(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if left.len() == 1 && right.len() == 1 { + match (left.get(0), right.get(0)) { + (Some(JsonValue::Number(l)), Some(JsonValue::Number(r))) => l.0 < r.0, + _ => false, + } + } else { + false + } +} + +/// compare elements +pub fn eq(left: Vec<&JsonValue>, right: Vec<&JsonValue>) -> bool { + if left.len() != right.len() { + false + } else { + left.iter().zip(right).map(|(a, b)| a.eq(&b)).all(|a| a) + } +} diff --git a/dozer-sql/src/jsonpath/path/mod.rs b/dozer-sql/src/jsonpath/path/mod.rs new file mode 100644 index 0000000000..8fc31f94c1 --- /dev/null +++ b/dozer-sql/src/jsonpath/path/mod.rs @@ -0,0 +1,76 @@ +use crate::jsonpath::parser::model::{Function, JsonPath, JsonPathIndex, Operand}; +use crate::jsonpath::path::index::{ArrayIndex, ArraySlice, Current, FilterPath, UnionIndex}; +use crate::jsonpath::path::top::{ + Chain, DescentObject, DescentWildcard, FnPath, IdentityPath, ObjectField, RootPointer, Wildcard, +}; +use crate::jsonpath::JsonPathValue; +use dozer_types::json_types::JsonValue; + +/// The module is in charge of processing [[JsonPathIndex]] elements +mod index; +/// The module is a helper module providing the set of helping funcitons to process a json elements +mod json; +/// The module is responsible for processing of the [[JsonPath]] elements +mod top; + +/// The trait defining the behaviour of processing every separated element. +/// type Data usually stands for json [[Value]] +/// The trait also requires to have a root json to process. +/// It needs in case if in the filter there will be a pointer to the absolute path +pub trait Path<'a> { + type Data; + /// when every element needs to handle independently + fn find(&self, input: JsonPathValue<'a, Self::Data>) -> Vec> { + vec![input] + } + /// when the whole output needs to handle + fn flat_find( + &self, + input: Vec>, + _is_search_length: bool, + ) -> Vec> { + input.into_iter().flat_map(|d| self.find(d)).collect() + } + /// defines when we need to invoke `find` or `flat_find` + fn needs_all(&self) -> bool { + false + } +} + +/// The basic type for instances. +pub type PathInstance<'a> = Box + 'a>; + +/// The major method to process the top part of json part +pub fn json_path_instance<'a>(json_path: &'a JsonPath, root: &'a JsonValue) -> PathInstance<'a> { + match json_path { + JsonPath::Root => Box::new(RootPointer::new(root)), + JsonPath::Field(key) => Box::new(ObjectField::new(key)), + JsonPath::Chain(chain) => Box::new(Chain::from(chain, root)), + JsonPath::Wildcard => Box::new(Wildcard {}), + JsonPath::Descent(key) => Box::new(DescentObject::new(key)), + JsonPath::DescentW => Box::new(DescentWildcard), + JsonPath::Current(value) => Box::new(Current::from(value, root)), + JsonPath::Index(index) => process_index(index, root), + JsonPath::Empty => Box::new(IdentityPath {}), + JsonPath::Fn(Function::Length) => Box::new(FnPath::Size), + } +} + +/// The method processes the indexes(all expressions indie []) +fn process_index<'a>(json_path_index: &'a JsonPathIndex, root: &'a JsonValue) -> PathInstance<'a> { + match json_path_index { + JsonPathIndex::Single(index) => Box::new(ArrayIndex::new(index.as_u64().unwrap() as usize)), + JsonPathIndex::Slice(s, e, step) => Box::new(ArraySlice::new(*s, *e, *step)), + JsonPathIndex::UnionKeys(elems) => Box::new(UnionIndex::from_keys(elems)), + JsonPathIndex::UnionIndex(elems) => Box::new(UnionIndex::from_indexes(elems)), + JsonPathIndex::Filter(fe) => Box::new(FilterPath::new(fe, root)), + } +} + +/// The method processes the operand inside the filter expressions +fn process_operand<'a>(op: &'a Operand, root: &'a JsonValue) -> PathInstance<'a> { + match op { + Operand::Static(v) => json_path_instance(&JsonPath::Root, v), + Operand::Dynamic(jp) => json_path_instance(jp, root), + } +} diff --git a/dozer-sql/src/jsonpath/path/top.rs b/dozer-sql/src/jsonpath/path/top.rs new file mode 100644 index 0000000000..65d2dc6714 --- /dev/null +++ b/dozer-sql/src/jsonpath/path/top.rs @@ -0,0 +1,314 @@ +use crate::jsonpath::parser::model::*; +use crate::jsonpath::path::JsonPathValue::{NewValue, NoValue, Slice}; +use crate::jsonpath::path::{json_path_instance, JsonPathValue, Path, PathInstance}; +use dozer_types::json_types::JsonValue::{Array, Object}; +use dozer_types::json_types::{serde_json_to_json_value, JsonValue}; +use dozer_types::serde_json::json; + +/// to process the element [*] +pub(crate) struct Wildcard {} + +impl<'a> Path<'a> for Wildcard { + type Data = JsonValue; + + fn find(&self, data: JsonPathValue<'a, Self::Data>) -> Vec> { + data.flat_map_slice(|data| { + let res = match data { + Array(elems) => { + let mut res = vec![]; + for el in elems.iter() { + res.push(Slice(el)); + } + + res + } + Object(elems) => { + let mut res = vec![]; + for el in elems.values() { + res.push(Slice(el)); + } + res + } + _ => vec![], + }; + if res.is_empty() { + vec![NoValue] + } else { + res + } + }) + } +} + +/// empty path. Returns incoming data. +pub(crate) struct IdentityPath {} + +impl<'a> Path<'a> for IdentityPath { + type Data = JsonValue; + + fn find(&self, data: JsonPathValue<'a, Self::Data>) -> Vec> { + vec![data] + } +} + +pub(crate) struct EmptyPath {} + +impl<'a> Path<'a> for EmptyPath { + type Data = JsonValue; + + fn find(&self, _data: JsonPathValue<'a, Self::Data>) -> Vec> { + vec![] + } +} + +/// process $ element +pub(crate) struct RootPointer<'a, T> { + root: &'a T, +} + +impl<'a, T> RootPointer<'a, T> { + pub(crate) fn new(root: &'a T) -> RootPointer<'a, T> { + RootPointer { root } + } +} + +impl<'a> Path<'a> for RootPointer<'a, JsonValue> { + type Data = JsonValue; + + fn find(&self, _data: JsonPathValue<'a, Self::Data>) -> Vec> { + vec![Slice(self.root)] + } +} + +/// process object fields like ['key'] or .key +pub(crate) struct ObjectField<'a> { + key: &'a str, +} + +impl<'a> ObjectField<'a> { + pub(crate) fn new(key: &'a str) -> ObjectField<'a> { + ObjectField { key } + } +} + +impl<'a> Clone for ObjectField<'a> { + fn clone(&self) -> Self { + ObjectField::new(self.key) + } +} + +impl<'a> Path<'a> for FnPath { + type Data = JsonValue; + + fn flat_find( + &self, + input: Vec>, + is_search_length: bool, + ) -> Vec> { + if JsonPathValue::only_no_value(&input) { + return vec![NoValue]; + } + + let res = if is_search_length { + NewValue( + serde_json_to_json_value(json!(input.iter().filter(|v| v.has_value()).count())) + .unwrap(), + ) + } else { + let take_len = |v: &JsonValue| match v { + Array(elems) => NewValue(serde_json_to_json_value(json!(elems.len())).unwrap()), + _ => NoValue, + }; + + match input.get(0) { + Some(v) => match v { + NewValue(d) => take_len(d), + Slice(s) => take_len(s), + NoValue => NoValue, + }, + None => NoValue, + } + }; + vec![res] + } + + fn needs_all(&self) -> bool { + true + } +} + +pub(crate) enum FnPath { + Size, +} + +impl<'a> Path<'a> for ObjectField<'a> { + type Data = JsonValue; + + fn find(&self, data: JsonPathValue<'a, Self::Data>) -> Vec> { + let take_field = |v: &'a JsonValue| match v { + Object(fields) => fields.get(self.key), + _ => None, + }; + + let res = match data { + Slice(js) => take_field(js).map(Slice).unwrap_or_else(|| NoValue), + _ => NoValue, + }; + vec![res] + } +} +/// the top method of the processing ..* +pub(crate) struct DescentWildcard; + +impl<'a> Path<'a> for DescentWildcard { + type Data = JsonValue; + + fn find(&self, data: JsonPathValue<'a, Self::Data>) -> Vec> { + data.map_slice(deep_flatten) + } +} + +fn deep_flatten(data: &JsonValue) -> Vec<&JsonValue> { + let mut acc = vec![]; + match data { + Object(elems) => { + for v in elems.values() { + acc.push(v); + acc.append(&mut deep_flatten(v)); + } + } + Array(elems) => { + for v in elems.iter() { + acc.push(v); + acc.append(&mut deep_flatten(v)); + } + } + _ => (), + } + acc +} + +fn deep_path_by_key<'a>(data: &'a JsonValue, key: ObjectField<'a>) -> Vec<&'a JsonValue> { + let mut level: Vec<&JsonValue> = JsonPathValue::into_data(key.find(data.into())); + match data { + Object(elems) => { + let mut next_levels: Vec<&JsonValue> = elems + .values() + .flat_map(|v| deep_path_by_key(v, key.clone())) + .collect(); + level.append(&mut next_levels); + level + } + Array(elems) => { + let mut next_levels: Vec<&JsonValue> = elems + .iter() + .flat_map(|v| deep_path_by_key(v, key.clone())) + .collect(); + level.append(&mut next_levels); + level + } + _ => level, + } +} + +/// processes decent object like .. +pub(crate) struct DescentObject<'a> { + key: &'a str, +} + +impl<'a> Path<'a> for DescentObject<'a> { + type Data = JsonValue; + + fn find(&self, data: JsonPathValue<'a, Self::Data>) -> Vec> { + data.flat_map_slice(|data| { + let res_col = deep_path_by_key(data, ObjectField::new(self.key)); + if res_col.is_empty() { + vec![NoValue] + } else { + JsonPathValue::map_vec(res_col) + } + }) + } +} + +impl<'a> DescentObject<'a> { + pub fn new(key: &'a str) -> Self { + DescentObject { key } + } +} + +/// the top method of the processing representing the chain of other operators +pub(crate) struct Chain<'a> { + chain: Vec>, + is_search_length: bool, +} + +impl<'a> Chain<'a> { + pub fn new(chain: Vec>, is_search_length: bool) -> Self { + Chain { + chain, + is_search_length, + } + } + pub fn from(chain: &'a [JsonPath], root: &'a JsonValue) -> Self { + let chain_len = chain.len(); + let is_search_length = if chain_len > 2 { + let mut res = false; + // if the result of the slice expected to be a slice, union or filter - + // length should return length of resulted array + // In all other cases, including single index, we should fetch item from resulting array + // and return length of that item + res = match chain.get(chain_len - 1).expect("chain element disappeared") { + JsonPath::Fn(Function::Length) => { + for item in chain.iter() { + match (item, res) { + // if we found union, slice, filter or wildcard - set search to true + ( + JsonPath::Index(JsonPathIndex::UnionIndex(_)) + | JsonPath::Index(JsonPathIndex::UnionKeys(_)) + | JsonPath::Index(JsonPathIndex::Slice(_, _, _)) + | JsonPath::Index(JsonPathIndex::Filter(_)) + | JsonPath::Wildcard, + false, + ) => { + res = true; + } + // if we found a fetching of single index - reset search to false + (JsonPath::Index(JsonPathIndex::Single(_)), true) => { + res = false; + } + (_, _) => {} + } + } + res + } + _ => false, + }; + res + } else { + false + }; + + Chain::new( + chain.iter().map(|p| json_path_instance(p, root)).collect(), + is_search_length, + ) + } +} + +impl<'a> Path<'a> for Chain<'a> { + type Data = JsonValue; + + fn find(&self, data: JsonPathValue<'a, Self::Data>) -> Vec> { + let mut res = vec![data]; + + for inst in self.chain.iter() { + if inst.needs_all() { + res = inst.flat_find(res, self.is_search_length) + } else { + res = res.into_iter().flat_map(|d| inst.find(d)).collect() + } + } + res + } +} diff --git a/dozer-sql/src/lib.rs b/dozer-sql/src/lib.rs index 32e4861968..88de44f389 100644 --- a/dozer-sql/src/lib.rs +++ b/dozer-sql/src/lib.rs @@ -3,4 +3,9 @@ extern crate core; // Re-export sqlparser pub use sqlparser; +pub mod jsonpath; pub mod pipeline; + +#[macro_use] +extern crate pest_derive; +extern crate pest; diff --git a/dozer-sql/src/pipeline/expression/builder.rs b/dozer-sql/src/pipeline/expression/builder.rs index 459575f698..09e830a955 100644 --- a/dozer-sql/src/pipeline/expression/builder.rs +++ b/dozer-sql/src/pipeline/expression/builder.rs @@ -22,6 +22,7 @@ use crate::pipeline::expression::execution::Expression::{ ConditionalExpression, GeoFunction, Now, ScalarFunction, }; use crate::pipeline::expression::geo::common::GeoFunctionType; +use crate::pipeline::expression::json_functions::JsonFunctionType; use crate::pipeline::expression::operator::{BinaryOperatorType, UnaryOperatorType}; use crate::pipeline::expression::scalar::common::ScalarFunctionType; use crate::pipeline::expression::scalar::string::TrimType; @@ -338,6 +339,27 @@ impl ExpressionBuilder { } } + fn json_func_check( + &mut self, + function_name: String, + parse_aggregations: bool, + sql_function: &Function, + schema: &Schema, + ) -> Result { + let mut function_args: Vec = Vec::new(); + for arg in &sql_function.args { + function_args.push(self.parse_sql_function_arg(parse_aggregations, arg, schema)?); + } + + match JsonFunctionType::new(function_name.as_str()) { + Ok(jft) => Ok(Expression::Json { + fun: jft, + args: function_args, + }), + Err(_e) => Err(InvalidFunction(function_name)), + } + } + fn conditional_expr_check( &mut self, function_name: String, @@ -414,7 +436,12 @@ impl ExpressionBuilder { return conditional_check; } - self.datetime_expr_check(function_name) + let datetime_check = self.datetime_expr_check(function_name.clone()); + if datetime_check.is_ok() { + return datetime_check; + } + + self.json_func_check(function_name, parse_aggregations, sql_function, schema) } fn parse_sql_function_arg( @@ -590,10 +617,9 @@ impl ExpressionBuilder { DataType::Timestamp(..) => CastOperatorType::Timestamp, DataType::Text => CastOperatorType::Text, DataType::String => CastOperatorType::String, + DataType::JSON => CastOperatorType::Json, DataType::Custom(name, ..) => { - if name.to_string().to_lowercase() == "json" { - CastOperatorType::Json - } else if name.to_string().to_lowercase() == "uint" { + if name.to_string().to_lowercase() == "uint" { CastOperatorType::UInt } else if name.to_string().to_lowercase() == "u128" { CastOperatorType::U128 diff --git a/dozer-sql/src/pipeline/expression/cast.rs b/dozer-sql/src/pipeline/expression/cast.rs index f1c2658327..c22924d1f7 100644 --- a/dozer-sql/src/pipeline/expression/cast.rs +++ b/dozer-sql/src/pipeline/expression/cast.rs @@ -178,7 +178,7 @@ impl CastOperatorType { } CastOperatorType::Json => { if let Some(value) = field.to_json() { - Ok(Field::Json(value.to_owned())) + Ok(Field::Json(value)) } else { Err(PipelineError::InvalidCast { from: field, @@ -202,6 +202,7 @@ impl CastOperatorType { FieldType::UInt, FieldType::I128, FieldType::U128, + FieldType::Json, ], FieldType::UInt, ), @@ -212,6 +213,7 @@ impl CastOperatorType { FieldType::UInt, FieldType::I128, FieldType::U128, + FieldType::Json, ], FieldType::U128, ), @@ -222,6 +224,7 @@ impl CastOperatorType { FieldType::UInt, FieldType::I128, FieldType::U128, + FieldType::Json, ], FieldType::Int, ), @@ -232,6 +235,7 @@ impl CastOperatorType { FieldType::UInt, FieldType::I128, FieldType::U128, + FieldType::Json, ], FieldType::I128, ), @@ -244,6 +248,7 @@ impl CastOperatorType { FieldType::String, FieldType::UInt, FieldType::U128, + FieldType::Json, ], FieldType::Float, ), @@ -256,6 +261,7 @@ impl CastOperatorType { FieldType::I128, FieldType::UInt, FieldType::U128, + FieldType::Json, ], FieldType::Boolean, ), @@ -273,6 +279,7 @@ impl CastOperatorType { FieldType::Timestamp, FieldType::UInt, FieldType::U128, + FieldType::Json, ], FieldType::String, ), @@ -290,6 +297,7 @@ impl CastOperatorType { FieldType::Timestamp, FieldType::UInt, FieldType::U128, + FieldType::Json, ], FieldType::Text, ), @@ -311,7 +319,20 @@ impl CastOperatorType { FieldType::Timestamp, ), CastOperatorType::Date => (vec![FieldType::Date, FieldType::String], FieldType::Date), - CastOperatorType::Json => (vec![FieldType::Json], FieldType::Json), + CastOperatorType::Json => ( + vec![ + FieldType::Boolean, + FieldType::Float, + FieldType::Int, + FieldType::I128, + FieldType::String, + FieldType::Text, + FieldType::UInt, + FieldType::U128, + FieldType::Json, + ], + FieldType::Json, + ), }; let expression_type = validate_arg_type(arg, expected_input_type, schema, self, 0)?; diff --git a/dozer-sql/src/pipeline/expression/execution.rs b/dozer-sql/src/pipeline/expression/execution.rs index 00db13e3fa..2da8b3c4e8 100644 --- a/dozer-sql/src/pipeline/expression/execution.rs +++ b/dozer-sql/src/pipeline/expression/execution.rs @@ -9,9 +9,11 @@ use crate::pipeline::expression::conditional::{ }; use crate::pipeline::expression::datetime::{get_datetime_function_type, DateTimeFunctionType}; use crate::pipeline::expression::geo::common::{get_geo_function_type, GeoFunctionType}; +use crate::pipeline::expression::json_functions::JsonFunctionType; use crate::pipeline::expression::operator::{BinaryOperatorType, UnaryOperatorType}; use crate::pipeline::expression::scalar::common::{get_scalar_function_type, ScalarFunctionType}; use crate::pipeline::expression::scalar::string::{evaluate_trim, validate_trim, TrimType}; + use dozer_types::types::{Field, FieldType, Record, Schema, SourceDefinition}; use uuid::Uuid; @@ -71,6 +73,10 @@ pub enum Expression { Now { fun: DateTimeFunctionType, }, + Json { + fun: JsonFunctionType, + args: Vec, + }, #[cfg(feature = "python")] PythonUDF { name: String, @@ -188,6 +194,17 @@ impl Expression { fun.to_string() + "(" + arg.to_string(schema).as_str() + ")" } Expression::Now { fun } => fun.to_string() + "()", + Expression::Json { fun, args } => { + fun.to_string() + + "(" + + args + .iter() + .map(|e| e.to_string(schema)) + .collect::>() + .join(",") + .as_str() + + ")" + } } } } @@ -266,6 +283,7 @@ impl ExpressionExecutor for Expression { Expression::ConditionalExpression { fun, args } => fun.evaluate(schema, args, record), Expression::DateTimeFunction { fun, arg } => fun.evaluate(schema, arg, record), Expression::Now { fun } => fun.evaluate_now(), + Expression::Json { fun, args } => fun.evaluate(schema, args, record), } } @@ -331,6 +349,12 @@ impl ExpressionExecutor for Expression { dozer_types::types::SourceDefinition::Dynamic, false, )), + Expression::Json { fun: _, args: _ } => Ok(ExpressionType::new( + FieldType::Json, + false, + dozer_types::types::SourceDefinition::Dynamic, + false, + )), #[cfg(feature = "python")] Expression::PythonUDF { return_type, .. } => Ok(ExpressionType::new( *return_type, diff --git a/dozer-sql/src/pipeline/expression/json_functions.rs b/dozer-sql/src/pipeline/expression/json_functions.rs new file mode 100644 index 0000000000..231aee9c7c --- /dev/null +++ b/dozer-sql/src/pipeline/expression/json_functions.rs @@ -0,0 +1,136 @@ +use crate::pipeline::errors::PipelineError; +use crate::pipeline::errors::PipelineError::{ + InvalidArgument, InvalidFunction, InvalidFunctionArgument, InvalidValue, +}; +use crate::pipeline::expression::execution::{Expression, ExpressionExecutor}; + +use crate::jsonpath::{JsonPathFinder, JsonPathInst}; +use dozer_types::json_types::JsonValue; +use dozer_types::types::{Field, Record, Schema}; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)] +pub enum JsonFunctionType { + JsonValue, + JsonQuery, +} + +impl Display for JsonFunctionType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + JsonFunctionType::JsonValue => f.write_str("JSON_VALUE".to_string().as_str()), + JsonFunctionType::JsonQuery => f.write_str("JSON_QUERY".to_string().as_str()), + } + } +} + +impl JsonFunctionType { + pub(crate) fn new(name: &str) -> Result { + match name { + "json_value" => Ok(JsonFunctionType::JsonValue), + "json_query" => Ok(JsonFunctionType::JsonQuery), + _ => Err(InvalidFunction(name.to_string())), + } + } + + pub(crate) fn evaluate( + &self, + schema: &Schema, + args: &Vec, + record: &Record, + ) -> Result { + match self { + JsonFunctionType::JsonValue => self.evaluate_json_value(schema, args, record), + JsonFunctionType::JsonQuery => self.evaluate_json_query(schema, args, record), + } + } + + pub(crate) fn evaluate_json_value( + &self, + schema: &Schema, + args: &Vec, + record: &Record, + ) -> Result { + if args.len() > 2 { + return Err(InvalidFunctionArgument( + self.to_string(), + args[2].evaluate(record, schema)?, + 2, + )); + } + let json_input = args[0].evaluate(record, schema)?; + let path = args[1] + .evaluate(record, schema)? + .to_string() + .ok_or(InvalidArgument(args[1].to_string(schema)))?; + + Ok(Field::Json(self.evaluate_json(json_input, path)?)) + } + + pub(crate) fn evaluate_json_query( + &self, + schema: &Schema, + args: &Vec, + record: &Record, + ) -> Result { + let mut path = String::from("$"); + if args.len() < 2 && !args.is_empty() { + Ok(Field::Json( + self.evaluate_json(args[0].evaluate(record, schema)?, path)?, + )) + } else if args.len() == 2 { + let json_input = args[0].evaluate(record, schema)?; + path = args[1] + .evaluate(record, schema)? + .to_string() + .ok_or(InvalidArgument(args[1].to_string(schema)))?; + + Ok(Field::Json(self.evaluate_json(json_input, path)?)) + } else { + Err(InvalidFunctionArgument( + self.to_string(), + args[2].evaluate(record, schema)?, + 2, + )) + } + } + + pub(crate) fn evaluate_json( + &self, + json_input: Field, + path: String, + ) -> Result { + let json_val = match json_input.to_json() { + Some(json) => json, + None => JsonValue::Null, + }; + + let finder = JsonPathFinder::new( + Box::from(json_val), + Box::from(JsonPathInst::from_str(path.as_str()).map_err(InvalidArgument)?), + ); + + match finder.find() { + JsonValue::Null => Ok(JsonValue::Null), + JsonValue::Array(a) => { + if a.is_empty() { + Ok(JsonValue::Array(vec![])) + } else if a.len() == 1 { + let item = match a.first() { + Some(i) => i, + None => return Err(InvalidValue("Invalid length of array".to_string())), + }; + Ok(item.to_owned()) + } else { + let mut array_val = vec![]; + for item in a { + array_val.push(item); + } + Ok(JsonValue::Array(array_val)) + } + } + _ => Err(InvalidValue(path)), + } + } +} diff --git a/dozer-sql/src/pipeline/expression/mod.rs b/dozer-sql/src/pipeline/expression/mod.rs index f7db367343..e02a26ceeb 100644 --- a/dozer-sql/src/pipeline/expression/mod.rs +++ b/dozer-sql/src/pipeline/expression/mod.rs @@ -7,6 +7,7 @@ pub mod conditional; mod datetime; pub mod execution; pub mod geo; +mod json_functions; pub mod logical; pub mod mathematical; pub mod operator; diff --git a/dozer-sql/src/pipeline/expression/tests/json_functions.rs b/dozer-sql/src/pipeline/expression/tests/json_functions.rs new file mode 100644 index 0000000000..8ecde666ad --- /dev/null +++ b/dozer-sql/src/pipeline/expression/tests/json_functions.rs @@ -0,0 +1,647 @@ +use crate::pipeline::expression::tests::test_common::run_fct; +use dozer_types::json_types::{serde_json_to_json_value, JsonValue}; +use dozer_types::ordered_float::OrderedFloat; +use dozer_types::serde_json::json; +use dozer_types::types::{Field, FieldDefinition, FieldType, Schema, SourceDefinition}; +use std::collections::BTreeMap; + +#[test] +fn test_json_value() { + let json_val = serde_json_to_json_value(json!( + { + "info":{ + "type":1, + "address":{ + "town":"Bristol", + "county":"Avon", + "country":"England" + }, + "tags":["Sport", "Water polo"] + }, + "type":"Basic" + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.info.address.town') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Json(JsonValue::String(String::from("Bristol")))); +} + +#[test] +fn test_json_value_null() { + let json_val = serde_json_to_json_value(json!( + { + "info":{ + "type":1, + "address":{ + "town":"Bristol", + "county":"Avon", + "country":"England" + }, + "tags":["Sport", "Water polo"] + }, + "type":"Basic" + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.info.address.tags') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Json(JsonValue::Null)); +} + +#[test] +fn test_json_query() { + let json_val = serde_json_to_json_value(json!( + { + "info": { + "type": 1, + "address": { + "town": "Cheltenham", + "county": "Gloucestershire", + "country": "England" + }, + "tags": ["Sport", "Water polo"] + }, + "type": "Basic" + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.info.address') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!( + f, + Field::Json(JsonValue::Object(BTreeMap::from([ + ( + "town".to_string(), + JsonValue::String("Cheltenham".to_string()) + ), + ( + "county".to_string(), + JsonValue::String("Gloucestershire".to_string()) + ), + ( + "country".to_string(), + JsonValue::String("England".to_string()) + ), + ]))) + ); +} + +#[test] +fn test_json_query_null() { + let json_val = serde_json_to_json_value(json!( + { + "info": { + "type": 1, + "address": { + "town": "Cheltenham", + "county": "Gloucestershire", + "country": "England" + }, + "tags": ["Sport", "Water polo"] + }, + "type": "Basic" + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.type') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + assert_eq!(f, Field::Json(JsonValue::String("Basic".to_string()))); +} + +#[test] +fn test_json_query_len_one_array() { + let json_val = serde_json_to_json_value(json!( + { + "info": { + "type": 1, + "address": { + "town": "Cheltenham", + "county": "Gloucestershire", + "country": "England" + }, + "tags": ["Sport"] + }, + "type": "Basic" + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_VALUE(jsonInfo,'$.info.tags') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + assert_eq!( + f, + Field::Json(JsonValue::Array(vec![JsonValue::String(String::from( + "Sport" + ))])) + ); +} + +#[test] +fn test_json_query_array() { + let json_val = serde_json_to_json_value(json!( + { + "info": { + "type": 1, + "address": { + "town": "Cheltenham", + "county": "Gloucestershire", + "country": "England" + }, + "tags": ["Sport", "Water polo"] + }, + "type": "Basic" + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_QUERY(jsonInfo,'$.info.tags') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!( + f, + Field::Json(JsonValue::Array(vec![ + JsonValue::String(String::from("Sport")), + JsonValue::String(String::from("Water polo")), + ])) + ); +} + +#[test] +fn test_json_query_default_path() { + let json_val = serde_json_to_json_value(json!( + { + "Cities": [ + { + "Name": "Kabul", + "CountryCode": "AFG", + "District": "Kabol", + "Population": 1780000 + }, + { + "Name": "Qandahar", + "CountryCode": "AFG", + "District": "Qandahar", + "Population": 237500 + } + ] + } + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_QUERY(jsonInfo) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + assert_eq!(f, Field::Json(json_val)); +} + +#[test] +fn test_json_query_all() { + let json_val = serde_json_to_json_value(json!( + [ + {"digit": 30, "letter": "A"}, + {"digit": 31, "letter": "B"} + ] + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_QUERY(jsonInfo, '$..*') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!( + f, + Field::Json( + serde_json_to_json_value(json!([ + { + "digit": 30, + "letter": "A" + }, + 30, + "A", + { + "digit": 31, + "letter": "B" + }, + 31, + "B" + ])) + .unwrap() + ) + ); +} + +#[test] +fn test_json_query_iter() { + let json_val = serde_json_to_json_value(json!( + [ + {"digit": 30, "letter": "A"}, + {"digit": 31, "letter": "B"} + ] + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_QUERY(jsonInfo, '$[*].digit') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!( + f, + Field::Json(serde_json_to_json_value(json!([30, 31,])).unwrap()) + ); +} + +#[test] +fn test_json_cast() { + let f = run_fct( + "SELECT CAST(uint AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("uint"), + FieldType::UInt, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::UInt(10_u64)], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(10_f64)))); + + let f = run_fct( + "SELECT CAST(u128 AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("u128"), + FieldType::U128, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::U128(10_u128)], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(10_f64)))); + + let f = run_fct( + "SELECT CAST(int AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("int"), + FieldType::Int, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Int(10_i64)], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(10_f64)))); + + let f = run_fct( + "SELECT CAST(i128 AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("i128"), + FieldType::I128, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::I128(10_i128)], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(10_f64)))); + + let f = run_fct( + "SELECT CAST(float AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("float"), + FieldType::Float, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Float(OrderedFloat(10_f64))], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(10_f64)))); + + let f = run_fct( + "SELECT CAST(str AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("str"), + FieldType::String, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::String("Dozer".to_string())], + ); + + assert_eq!(f, Field::Json(JsonValue::String("Dozer".to_string()))); + + let f = run_fct( + "SELECT CAST(str AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("str"), + FieldType::Text, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Text("Dozer".to_string())], + ); + + assert_eq!(f, Field::Json(JsonValue::String("Dozer".to_string()))); + + let f = run_fct( + "SELECT CAST(bool AS JSON) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("bool"), + FieldType::Boolean, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Boolean(true)], + ); + + assert_eq!(f, Field::Json(JsonValue::Bool(true))); +} + +#[test] +fn test_json_value_cast() { + let json_val = serde_json_to_json_value(json!( + [ + {"digit": 30, "letter": "A"}, + {"digit": 31, "letter": "B"} + ] + )) + .unwrap(); + + let f = run_fct( + "SELECT JSON_VALUE(jsonInfo, '$[0].digit') FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Json(JsonValue::Number(OrderedFloat(30_f64)))); + + let f = run_fct( + "SELECT CAST(JSON_VALUE(jsonInfo, '$[0].digit') AS UINT) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::UInt(30_u64)); + + let f = run_fct( + "SELECT CAST(JSON_VALUE(jsonInfo, '$[0].digit') AS INT) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Int(30_i64)); + + let f = run_fct( + "SELECT CAST(JSON_VALUE(jsonInfo, '$[0].digit') AS FLOAT) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val.clone())], + ); + + assert_eq!(f, Field::Float(OrderedFloat(30_f64))); + + let f = run_fct( + "SELECT CAST(JSON_VALUE(jsonInfo, '$[0].digit') AS STRING) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::String("30".to_string())); + + let json_val = serde_json_to_json_value(json!( + [ + {"bool": true}, + {"digit": 31, "letter": "B"} + ] + )) + .unwrap(); + + let f = run_fct( + "SELECT CAST(JSON_VALUE(jsonInfo, '$[0].bool') AS BOOLEAN) FROM users", + Schema::empty() + .field( + FieldDefinition::new( + String::from("jsonInfo"), + FieldType::Json, + false, + SourceDefinition::Dynamic, + ), + false, + ) + .clone(), + vec![Field::Json(json_val)], + ); + + assert_eq!(f, Field::Boolean(true)); +} diff --git a/dozer-sql/src/pipeline/expression/tests/mod.rs b/dozer-sql/src/pipeline/expression/tests/mod.rs index ad59105cdd..baeaa7d500 100644 --- a/dozer-sql/src/pipeline/expression/tests/mod.rs +++ b/dozer-sql/src/pipeline/expression/tests/mod.rs @@ -14,6 +14,8 @@ mod datetime; #[cfg(test)] mod distance; #[cfg(test)] +mod json_functions; +#[cfg(test)] mod logical; #[cfg(test)] mod mathematical; diff --git a/dozer-types/src/json_types.rs b/dozer-types/src/json_types.rs index fa52485797..814f54adde 100644 --- a/dozer-types/src/json_types.rs +++ b/dozer-types/src/json_types.rs @@ -4,10 +4,11 @@ use chrono::SecondsFormat; use ordered_float::OrderedFloat; use prost_types::value::Kind; use prost_types::{ListValue, Struct, Value as ProstValue}; +use rust_decimal::prelude::FromPrimitive; use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; +use serde_json::{json as serde_json, Map, Value}; +use std::borrow::Cow; use std::collections::BTreeMap; - use std::fmt::{Display, Formatter}; use std::str::FromStr; @@ -21,6 +22,85 @@ pub enum JsonValue { Object(BTreeMap), } +#[allow(clippy::derivable_impls)] +impl Default for JsonValue { + fn default() -> JsonValue { + JsonValue::Null + } +} + +impl JsonValue { + pub fn as_array(&self) -> Option<&Vec> { + match self { + JsonValue::Array(array) => Some(array), + _ => None, + } + } + + pub fn as_object(&self) -> Option<&BTreeMap> { + match self { + JsonValue::Object(map) => Some(map), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + JsonValue::String(s) => Some(s), + _ => None, + } + } + + pub fn as_i64(&self) -> Option { + match self { + JsonValue::Number(n) => Some(n.0 as i64), + _ => None, + } + } + + pub fn as_i128(&self) -> Option { + match self { + JsonValue::Number(n) => i128::from_f64(n.0), + _ => None, + } + } + + pub fn as_u64(&self) -> Option { + match self { + JsonValue::Number(n) => Some(n.0 as u64), + _ => None, + } + } + + pub fn as_u128(&self) -> Option { + match self { + JsonValue::Number(n) => u128::from_f64(n.0), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + JsonValue::Number(n) => Some(n.0), + _ => None, + } + } + + pub fn as_bool(&self) -> Option { + match *self { + JsonValue::Bool(b) => Some(b), + _ => None, + } + } + + pub fn as_null(&self) -> Option<()> { + match *self { + JsonValue::Null => Some(()), + _ => None, + } + } +} + impl Display for JsonValue { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { @@ -46,8 +126,124 @@ impl FromStr for JsonValue { type Err = DeserializationError; fn from_str(s: &str) -> Result { - let object: Value = serde_json::from_str(s)?; - serde_json_to_json_value(object) + let object = serde_json::from_str(s); + if object.is_ok() { + serde_json_to_json_value(object?) + } else { + let f = OrderedFloat::from_str(s) + .map_err(|e| DeserializationError::Custom(Box::from(format!("{:?}", e)))); + if f.is_ok() { + Ok(JsonValue::Number(f?)) + } else { + let b = bool::from_str(s) + .map_err(|e| DeserializationError::Custom(Box::from(format!("{:?}", e)))); + if b.is_ok() { + Ok(JsonValue::Bool(b?)) + } else { + Ok(JsonValue::String(String::from(s))) + } + } + } + } +} + +impl From for JsonValue { + fn from(f: usize) -> Self { + From::from(f as f64) + } +} + +impl From for JsonValue { + fn from(f: f32) -> Self { + From::from(f as f64) + } +} + +impl From for JsonValue { + fn from(f: f64) -> Self { + JsonValue::Number(OrderedFloat(f)) + } +} + +impl From for JsonValue { + fn from(f: bool) -> Self { + JsonValue::Bool(f) + } +} + +impl From for JsonValue { + fn from(f: String) -> Self { + JsonValue::String(f) + } +} + +impl<'a> From<&'a str> for JsonValue { + fn from(f: &str) -> Self { + JsonValue::String(f.to_string()) + } +} + +impl<'a> From> for JsonValue { + fn from(f: Cow<'a, str>) -> Self { + JsonValue::String(f.into_owned()) + } +} + +impl From> for JsonValue { + fn from(f: OrderedFloat) -> Self { + JsonValue::Number(f) + } +} + +impl From> for JsonValue { + fn from(f: BTreeMap) -> Self { + JsonValue::Object(f) + } +} + +impl> From> for JsonValue { + fn from(f: Vec) -> Self { + JsonValue::Array(f.into_iter().map(Into::into).collect()) + } +} + +impl<'a, T: Clone + Into> From<&'a [T]> for JsonValue { + fn from(f: &'a [T]) -> Self { + JsonValue::Array(f.iter().cloned().map(Into::into).collect()) + } +} + +impl> FromIterator for JsonValue { + fn from_iter>(iter: I) -> Self { + JsonValue::Array(iter.into_iter().map(Into::into).collect()) + } +} + +impl, V: Into> FromIterator<(K, V)> for JsonValue { + fn from_iter>(iter: I) -> Self { + JsonValue::Object( + iter.into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + ) + } +} + +impl From<()> for JsonValue { + fn from((): ()) -> Self { + JsonValue::Null + } +} + +impl From> for JsonValue +where + T: Into, +{ + fn from(opt: Option) -> Self { + match opt { + None => JsonValue::Null, + Some(value) => Into::into(value), + } } } @@ -95,7 +291,7 @@ pub fn json_value_to_serde_json(value: JsonValue) -> Result Ok(Value::Bool(b)), JsonValue::Number(n) => { if n.0.is_finite() { - Ok(json!(n.0)) + Ok(serde_json!(n.0)) } else { Err(DeserializationError::F64TypeConversionError) } diff --git a/dozer-types/src/types/field.rs b/dozer-types/src/types/field.rs index b6669a131d..1d8545fd24 100644 --- a/dozer-types/src/types/field.rs +++ b/dozer-types/src/types/field.rs @@ -184,6 +184,7 @@ impl Field { pub fn as_uint(&self) -> Option { match self { Field::UInt(i) => Some(*i), + Field::Json(JsonValue::Number(f)) => Some(f.0 as u64), _ => None, } } @@ -191,6 +192,7 @@ impl Field { pub fn as_u128(&self) -> Option { match self { Field::U128(i) => Some(*i), + Field::Json(j) => j.as_u128(), _ => None, } } @@ -198,6 +200,7 @@ impl Field { pub fn as_int(&self) -> Option { match self { Field::Int(i) => Some(*i), + Field::Json(j) => j.as_i64(), _ => None, } } @@ -205,6 +208,7 @@ impl Field { pub fn as_i128(&self) -> Option { match self { Field::I128(i) => Some(*i), + Field::Json(j) => j.as_i128(), _ => None, } } @@ -212,6 +216,7 @@ impl Field { pub fn as_float(&self) -> Option { match self { Field::Float(f) => Some(f.0), + Field::Json(j) => j.as_f64(), _ => None, } } @@ -219,6 +224,7 @@ impl Field { pub fn as_boolean(&self) -> Option { match self { Field::Boolean(b) => Some(*b), + Field::Json(j) => j.as_bool(), _ => None, } } @@ -226,6 +232,7 @@ impl Field { pub fn as_string(&self) -> Option<&str> { match self { Field::String(s) => Some(s), + Field::Json(j) => j.as_str(), _ => None, } } @@ -233,6 +240,7 @@ impl Field { pub fn as_text(&self) -> Option<&str> { match self { Field::Text(s) => Some(s), + Field::Json(j) => j.as_str(), _ => None, } } @@ -305,6 +313,7 @@ impl Field { pub fn as_null(&self) -> Option<()> { match self { Field::Null => Some(()), + Field::Json(j) => j.as_null(), _ => None, } } @@ -319,6 +328,12 @@ impl Field { Field::Decimal(d) => d.to_u64(), Field::String(s) => s.parse::().ok(), Field::Text(s) => s.parse::().ok(), + Field::Json(j) => match j { + &JsonValue::Number(n) => u64::from_f64(*n), + JsonValue::String(s) => s.parse::().ok(), + &JsonValue::Null => Some(0_u64), + _ => None, + }, Field::Null => Some(0_u64), _ => None, } @@ -334,6 +349,12 @@ impl Field { Field::Decimal(d) => d.to_u128(), Field::String(s) => s.parse::().ok(), Field::Text(s) => s.parse::().ok(), + Field::Json(j) => match j { + &JsonValue::Number(n) => u128::from_f64(*n), + JsonValue::String(s) => s.parse::().ok(), + &JsonValue::Null => Some(0_u128), + _ => None, + }, Field::Null => Some(0_u128), _ => None, } @@ -349,6 +370,12 @@ impl Field { Field::Decimal(d) => d.to_i64(), Field::String(s) => s.parse::().ok(), Field::Text(s) => s.parse::().ok(), + Field::Json(j) => match j { + &JsonValue::Number(n) => i64::from_f64(*n), + JsonValue::String(s) => s.parse::().ok(), + &JsonValue::Null => Some(0_i64), + _ => None, + }, Field::Null => Some(0_i64), _ => None, } @@ -364,6 +391,12 @@ impl Field { Field::Decimal(d) => d.to_i128(), Field::String(s) => s.parse::().ok(), Field::Text(s) => s.parse::().ok(), + Field::Json(j) => match j { + &JsonValue::Number(n) => i128::from_f64(*n), + JsonValue::String(s) => s.parse::().ok(), + &JsonValue::Null => Some(0_i128), + _ => None, + }, Field::Null => Some(0_i128), _ => None, } @@ -379,6 +412,12 @@ impl Field { Field::Decimal(d) => d.to_f64(), Field::String(s) => s.parse::().ok(), Field::Text(s) => s.parse::().ok(), + Field::Json(j) => match j { + &JsonValue::Number(n) => Some(*n), + JsonValue::String(s) => s.parse::().ok(), + &JsonValue::Null => Some(0_f64), + _ => None, + }, Field::Null => Some(0_f64), _ => None, } @@ -395,6 +434,11 @@ impl Field { Field::Boolean(b) => Some(*b), Field::String(s) => s.parse::().ok(), Field::Text(s) => s.parse::().ok(), + Field::Json(j) => match j { + JsonValue::Bool(b) => Some(*b), + JsonValue::Null => Some(false), + _ => None, + }, Field::Null => Some(false), _ => None, } @@ -418,6 +462,7 @@ impl Field { Field::Date(d) => Some(d.format("%Y-%m-%d").to_string()), Field::Timestamp(t) => Some(t.to_rfc3339()), Field::Binary(b) => Some(format!("{b:X?}")), + Field::Json(j) => Some(j.to_string()), Field::Null => Some("".to_string()), _ => None, } @@ -441,6 +486,7 @@ impl Field { Field::Date(d) => Some(d.format("%Y-%m-%d").to_string()), Field::Timestamp(t) => Some(t.to_rfc3339()), Field::Binary(b) => Some(format!("{b:X?}")), + Field::Json(j) => Some(j.to_string()), Field::Null => Some("".to_string()), _ => None, } @@ -493,9 +539,21 @@ impl Field { } } - pub fn to_json(&self) -> Option<&JsonValue> { + pub fn to_json(&self) -> Option { match self { - Field::Json(b) => Some(b), + Field::Json(b) => Some(b.to_owned()), + Field::UInt(u) => Some(JsonValue::Number(OrderedFloat((*u) as f64))), + Field::U128(u) => Some(JsonValue::Number(OrderedFloat((*u) as f64))), + Field::Int(i) => Some(JsonValue::Number(OrderedFloat((*i) as f64))), + Field::I128(i) => Some(JsonValue::Number(OrderedFloat((*i) as f64))), + Field::Float(f) => Some(JsonValue::Number(*f)), + Field::Boolean(b) => Some(JsonValue::Bool(*b)), + Field::String(s) => match JsonValue::from_str(s.as_str()) { + Ok(v) => Some(v), + _ => None, + }, + Field::Text(t) => Some(JsonValue::String(t.to_owned())), + Field::Null => Some(JsonValue::Null), _ => None, } } diff --git a/dozer-types/src/types/tests.rs b/dozer-types/src/types/tests.rs index 73d5aef765..efc3726b64 100644 --- a/dozer-types/src/types/tests.rs +++ b/dozer-types/src/types/tests.rs @@ -321,7 +321,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_some()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_ok()); assert!(field.to_null().is_none()); @@ -339,7 +339,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_some()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_ok()); assert!(field.to_null().is_none()); @@ -357,7 +357,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_some()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_ok()); assert!(field.to_null().is_none()); @@ -375,7 +375,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_some()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_ok()); assert!(field.to_null().is_none()); @@ -393,7 +393,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_some()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_err()); assert!(field.to_null().is_none()); @@ -411,7 +411,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_none()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_err()); assert!(field.to_null().is_none()); @@ -429,7 +429,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_none()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_err()); assert!(field.to_null().is_none()); @@ -447,7 +447,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_none()); assert!(field.to_timestamp().unwrap().is_none()); assert!(field.to_date().unwrap().is_none()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_err()); assert!(field.to_null().is_none()); @@ -531,8 +531,8 @@ fn test_to_conversion() { assert!(field.to_i128().is_none()); assert!(field.to_float().is_none()); assert!(field.to_boolean().is_none()); - assert!(field.to_string().is_none()); - assert!(field.to_text().is_none()); + assert!(field.to_string().is_some()); + assert!(field.to_text().is_some()); assert!(field.to_binary().is_none()); assert!(field.to_decimal().is_none()); assert!(field.to_timestamp().unwrap().is_none()); @@ -594,7 +594,7 @@ fn test_to_conversion() { assert!(field.to_decimal().is_some()); assert!(field.to_timestamp().unwrap().is_some()); assert!(field.to_date().unwrap().is_some()); - assert!(field.to_json().is_none()); + assert!(field.to_json().is_some()); assert!(field.to_point().is_none()); assert!(field.to_duration().is_ok()); assert!(field.to_null().is_some());