Change to virtual workspace with tiledb-api crate, and add tiledb-arr…

…ow crate (#8) * Virtual workspace with tiledb-api, tiledb-sys crates * Add tiledb-arrow crate * arrow_type_physical with proptest * Proper directory structure, all crates in 'tiledb'
TileDB-Inc · Mar 19, 2024 · 78a073e · 78a073e
1 parent 35ff669
commit 78a073e
Show file tree

Hide file tree

Showing 50 changed files with 216 additions and 12 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,10 +1,12 @@
-[package]
-name = "tiledb"
-version = "0.1.0"
-edition = "2021"
+[workspace]
+members = [ "tiledb/api", "tiledb/arrow", "tiledb/sys" ]
+resolver = "2"
 
-[dependencies]
-tiledb-sys = {version = "0.1.0", path = "tiledb-sys" }
+[workspace.package]
+edition = "2021"
+rust-version = "1.72"
+version = "0.1.0"
 
-[dev-dependencies]
-tempdir = "0.3.7"
+[workspace.dependencies]
+tiledb = { path = "tiledb/api", version = "0.1.0" }
+tiledb-sys = { path = "tiledb/sys", version = "0.1.0" }
diff --git a/tiledb-sys/Cargo.toml b/tiledb-sys/Cargo.toml
diff --git a/tiledb/api/Cargo.toml b/tiledb/api/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "tiledb"
+version = { workspace = true }
+edition = { workspace = true }
+
+[lib]
+name = "tiledb"
+path = "src/lib.rs"
+
+[dependencies]
+tiledb-sys = { workspace = true }
+
+[dev-dependencies]
+tempdir = "0.3.7"
diff --git a/examples/quickstart_dense.rs → tiledb/api/examples/quickstart_dense.rs b/examples/quickstart_dense.rs → tiledb/api/examples/quickstart_dense.rs
diff --git a/src/array/attribute.rs → tiledb/api/src/array/attribute.rs b/src/array/attribute.rs → tiledb/api/src/array/attribute.rs
diff --git a/src/array/dimension.rs → tiledb/api/src/array/dimension.rs b/src/array/dimension.rs → tiledb/api/src/array/dimension.rs
diff --git a/src/array/domain.rs → tiledb/api/src/array/domain.rs b/src/array/domain.rs → tiledb/api/src/array/domain.rs
diff --git a/src/array/mod.rs → tiledb/api/src/array/mod.rs b/src/array/mod.rs → tiledb/api/src/array/mod.rs
diff --git a/src/array/schema.rs → tiledb/api/src/array/schema.rs b/src/array/schema.rs → tiledb/api/src/array/schema.rs
diff --git a/src/config.rs → tiledb/api/src/config.rs b/src/config.rs → tiledb/api/src/config.rs
diff --git a/src/context.rs → tiledb/api/src/context.rs b/src/context.rs → tiledb/api/src/context.rs
diff --git a/src/convert.rs → tiledb/api/src/convert.rs b/src/convert.rs → tiledb/api/src/convert.rs
diff --git a/src/error.rs → tiledb/api/src/error.rs b/src/error.rs → tiledb/api/src/error.rs
diff --git a/src/filter.rs → tiledb/api/src/filter.rs b/src/filter.rs → tiledb/api/src/filter.rs
diff --git a/src/filter_list.rs → tiledb/api/src/filter_list.rs b/src/filter_list.rs → tiledb/api/src/filter_list.rs
diff --git a/src/lib.rs → tiledb/api/src/lib.rs b/src/lib.rs → tiledb/api/src/lib.rs
diff --git a/src/query/mod.rs → tiledb/api/src/query/mod.rs b/src/query/mod.rs → tiledb/api/src/query/mod.rs
diff --git a/src/query/subarray.rs → tiledb/api/src/query/subarray.rs b/src/query/subarray.rs → tiledb/api/src/query/subarray.rs
diff --git a/src/string.rs → tiledb/api/src/string.rs b/src/string.rs → tiledb/api/src/string.rs
diff --git a/tests/enum_conversion_tests.rs → tiledb/api/tests/enum_conversion_tests.rs b/tests/enum_conversion_tests.rs → tiledb/api/tests/enum_conversion_tests.rs
diff --git a/tiledb/arrow/Cargo.toml b/tiledb/arrow/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "tiledb-arrow"
+version = { workspace = true }
+edition = { workspace = true }
+
+[dependencies]
+arrow = { version = "50.0.0", features = ["prettyprint"] }
+arrow-schema = { version = "50.0.0", default-features = false }
+tiledb = { workspace = true }
+
+[dev-dependencies]
+proptest = "1.0.0"
diff --git a/tiledb/arrow/src/datatype.rs b/tiledb/arrow/src/datatype.rs
@@ -0,0 +1,168 @@
+/// For a TileDB type, returns an Arrow type if the bits of the canonical input type match.
+/// If this returns Some(arrow_dt), then values of arrow_dt can be used in functions which expect tdb_dt, and vice verse.
+pub fn arrow_type_physical(
+    tdb_dt: &tiledb::Datatype,
+) -> Option<arrow_schema::DataType> {
+    match *tdb_dt {
+        tiledb::Datatype::Int8 => Some(arrow_schema::DataType::Int8),
+        tiledb::Datatype::Int16 => Some(arrow_schema::DataType::Int16),
+        tiledb::Datatype::Int32 => Some(arrow_schema::DataType::Int32),
+        tiledb::Datatype::Int64 => Some(arrow_schema::DataType::Int64),
+        tiledb::Datatype::UInt8 => Some(arrow_schema::DataType::UInt8),
+        tiledb::Datatype::UInt16 => Some(arrow_schema::DataType::UInt16),
+        tiledb::Datatype::UInt32 => Some(arrow_schema::DataType::UInt32),
+        tiledb::Datatype::UInt64 => Some(arrow_schema::DataType::UInt64),
+        tiledb::Datatype::Float32 => Some(arrow_schema::DataType::Float32),
+        tiledb::Datatype::Float64 => Some(arrow_schema::DataType::Float64),
+        tiledb::Datatype::Char => None,
+        tiledb::Datatype::StringAscii => None,
+        tiledb::Datatype::StringUtf8 => None,
+        tiledb::Datatype::StringUtf16 => None,
+        tiledb::Datatype::StringUtf32 => None,
+        tiledb::Datatype::StringUcs2 => None,
+        tiledb::Datatype::StringUcs4 => None,
+        tiledb::Datatype::Any => None,
+        tiledb::Datatype::DateTimeYear => None,
+        tiledb::Datatype::DateTimeMonth => None,
+        tiledb::Datatype::DateTimeWeek => None,
+        tiledb::Datatype::DateTimeDay => None,
+        tiledb::Datatype::DateTimeHour => None,
+        tiledb::Datatype::DateTimeMinute => None,
+        tiledb::Datatype::DateTimeSecond => {
+            Some(arrow_schema::DataType::Timestamp(
+                arrow_schema::TimeUnit::Second,
+                None,
+            ))
+        }
+        tiledb::Datatype::DateTimeMillisecond => {
+            Some(arrow_schema::DataType::Timestamp(
+                arrow_schema::TimeUnit::Millisecond,
+                None,
+            ))
+        }
+        tiledb::Datatype::DateTimeMicrosecond => {
+            Some(arrow_schema::DataType::Timestamp(
+                arrow_schema::TimeUnit::Microsecond,
+                None,
+            ))
+        }
+        tiledb::Datatype::DateTimeNanosecond => {
+            Some(arrow_schema::DataType::Timestamp(
+                arrow_schema::TimeUnit::Microsecond,
+                None,
+            ))
+        }
+        tiledb::Datatype::DateTimePicosecond => None,
+        tiledb::Datatype::DateTimeFemtosecond => None,
+        tiledb::Datatype::DateTimeAttosecond => None,
+        tiledb::Datatype::TimeHour => None,
+        tiledb::Datatype::TimeMinute => None,
+        tiledb::Datatype::TimeSecond => None, // TODO: arrow type is 32 bits, is tiledb type?
+        tiledb::Datatype::TimeMillisecond => None,
+        tiledb::Datatype::TimeMicrosecond => Some(
+            arrow_schema::DataType::Time64(arrow_schema::TimeUnit::Microsecond),
+        ),
+        tiledb::Datatype::TimeNanosecond => Some(
+            arrow_schema::DataType::Time64(arrow_schema::TimeUnit::Nanosecond),
+        ),
+        tiledb::Datatype::TimePicosecond => None,
+        tiledb::Datatype::TimeFemtosecond => None,
+        tiledb::Datatype::TimeAttosecond => None,
+        tiledb::Datatype::Blob => None,
+        tiledb::Datatype::Boolean => None,
+        tiledb::Datatype::GeometryWkb => None,
+        tiledb::Datatype::GeometryWkt => None,
+    }
+}
+
+/// For an Arrow type, returns a TileDB type if the bits of the canonical input type match.
+/// If this returns Some(tdb_t), then values for tdb_t can be used in functions which expect
+/// arrow_dt and vice verse.
+pub fn tiledb_type_physical(
+    arrow_dt: &arrow_schema::DataType,
+) -> Option<tiledb::Datatype> {
+    match *arrow_dt {
+        arrow_schema::DataType::Int8 => Some(tiledb::Datatype::Int8),
+        arrow_schema::DataType::Int16 => Some(tiledb::Datatype::Int16),
+        arrow_schema::DataType::Int32 => Some(tiledb::Datatype::Int32),
+        arrow_schema::DataType::Int64 => Some(tiledb::Datatype::Int64),
+        arrow_schema::DataType::UInt8 => Some(tiledb::Datatype::UInt8),
+        arrow_schema::DataType::UInt16 => Some(tiledb::Datatype::UInt16),
+        arrow_schema::DataType::UInt32 => Some(tiledb::Datatype::UInt32),
+        arrow_schema::DataType::UInt64 => Some(tiledb::Datatype::UInt64),
+        arrow_schema::DataType::Float32 => Some(tiledb::Datatype::Float32),
+        arrow_schema::DataType::Float64 => Some(tiledb::Datatype::Float64),
+        _ => None, // TODO
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+
+    fn datatype_strategy() -> impl Strategy<Value = tiledb::Datatype> {
+        prop_oneof![
+            Just(tiledb::Datatype::Int8),
+            Just(tiledb::Datatype::Int16),
+            Just(tiledb::Datatype::Int32),
+            Just(tiledb::Datatype::Int64),
+            Just(tiledb::Datatype::UInt8),
+            Just(tiledb::Datatype::UInt16),
+            Just(tiledb::Datatype::UInt32),
+            Just(tiledb::Datatype::UInt64),
+            Just(tiledb::Datatype::Float32),
+            Just(tiledb::Datatype::Float64),
+            Just(tiledb::Datatype::Char),
+            Just(tiledb::Datatype::StringAscii),
+            Just(tiledb::Datatype::StringUtf8),
+            Just(tiledb::Datatype::StringUtf16),
+            Just(tiledb::Datatype::StringUtf32),
+            Just(tiledb::Datatype::StringUcs2),
+            Just(tiledb::Datatype::StringUcs4),
+            Just(tiledb::Datatype::Any),
+            Just(tiledb::Datatype::DateTimeYear),
+            Just(tiledb::Datatype::DateTimeMonth),
+            Just(tiledb::Datatype::DateTimeWeek),
+            Just(tiledb::Datatype::DateTimeDay),
+            Just(tiledb::Datatype::DateTimeHour),
+            Just(tiledb::Datatype::DateTimeMinute),
+            Just(tiledb::Datatype::DateTimeSecond),
+            Just(tiledb::Datatype::DateTimeMillisecond),
+            Just(tiledb::Datatype::DateTimeMicrosecond),
+            Just(tiledb::Datatype::DateTimeNanosecond),
+            Just(tiledb::Datatype::DateTimePicosecond),
+            Just(tiledb::Datatype::DateTimeFemtosecond),
+            Just(tiledb::Datatype::DateTimeAttosecond),
+            Just(tiledb::Datatype::TimeHour),
+            Just(tiledb::Datatype::TimeMinute),
+            Just(tiledb::Datatype::TimeSecond),
+            Just(tiledb::Datatype::TimeMillisecond),
+            Just(tiledb::Datatype::TimeMicrosecond),
+            Just(tiledb::Datatype::TimeNanosecond),
+            Just(tiledb::Datatype::TimePicosecond),
+            Just(tiledb::Datatype::TimeFemtosecond),
+            Just(tiledb::Datatype::TimeAttosecond),
+            Just(tiledb::Datatype::Blob),
+            Just(tiledb::Datatype::Boolean),
+            Just(tiledb::Datatype::GeometryWkb),
+            Just(tiledb::Datatype::GeometryWkt),
+        ]
+    }
+
+    proptest! {
+        #[test]
+        fn test_physical(tdb_dt in datatype_strategy()) {
+            if let Some(arrow_dt) = arrow_type_physical(&tdb_dt) {
+                if let Some(adt_width) = arrow_dt.primitive_width() {
+                    let tdb_width : usize = tdb_dt.size().try_into().unwrap();
+                    assert_eq!(adt_width, tdb_width);
+                } else {
+                    // TODO: assert that `tdb_dt` is variable-length
+                }
+
+                // TODO: invertibility
+            }
+        }
+    }
+}
diff --git a/tiledb/arrow/src/lib.rs b/tiledb/arrow/src/lib.rs
@@ -0,0 +1,8 @@
+extern crate arrow;
+extern crate arrow_schema;
+extern crate tiledb;
+
+#[cfg(test)]
+extern crate proptest;
+
+pub mod datatype;
diff --git a/tiledb-sys/.gitignore → tiledb/sys/.gitignore b/tiledb-sys/.gitignore → tiledb/sys/.gitignore
diff --git a/tiledb/sys/Cargo.toml b/tiledb/sys/Cargo.toml
@@ -0,0 +1,4 @@
+[package]
+name = "tiledb-sys"
+version = { workspace = true }
+edition = { workspace = true }
diff --git a/tiledb-sys/Makefile → tiledb/sys/Makefile b/tiledb-sys/Makefile → tiledb/sys/Makefile
diff --git a/tiledb-sys/build.rs → tiledb/sys/build.rs b/tiledb-sys/build.rs → tiledb/sys/build.rs
diff --git a/tiledb-sys/src/array.rs → tiledb/sys/src/array.rs b/tiledb-sys/src/array.rs → tiledb/sys/src/array.rs
diff --git a/tiledb-sys/src/attribute.rs → tiledb/sys/src/attribute.rs b/tiledb-sys/src/attribute.rs → tiledb/sys/src/attribute.rs
diff --git a/tiledb-sys/src/capi_enum.rs → tiledb/sys/src/capi_enum.rs b/tiledb-sys/src/capi_enum.rs → tiledb/sys/src/capi_enum.rs
diff --git a/tiledb-sys/src/config.rs → tiledb/sys/src/config.rs b/tiledb-sys/src/config.rs → tiledb/sys/src/config.rs
diff --git a/tiledb-sys/src/constants.rs → tiledb/sys/src/constants.rs b/tiledb-sys/src/constants.rs → tiledb/sys/src/constants.rs
diff --git a/tiledb-sys/src/context.rs → tiledb/sys/src/context.rs b/tiledb-sys/src/context.rs → tiledb/sys/src/context.rs
diff --git a/tiledb-sys/src/datatype.rs → tiledb/sys/src/datatype.rs b/tiledb-sys/src/datatype.rs → tiledb/sys/src/datatype.rs
diff --git a/tiledb-sys/src/dimension.rs → tiledb/sys/src/dimension.rs b/tiledb-sys/src/dimension.rs → tiledb/sys/src/dimension.rs
diff --git a/tiledb-sys/src/domain.rs → tiledb/sys/src/domain.rs b/tiledb-sys/src/domain.rs → tiledb/sys/src/domain.rs
diff --git a/tiledb-sys/src/error.rs → tiledb/sys/src/error.rs b/tiledb-sys/src/error.rs → tiledb/sys/src/error.rs
diff --git a/tiledb-sys/src/filesystem.rs → tiledb/sys/src/filesystem.rs b/tiledb-sys/src/filesystem.rs → tiledb/sys/src/filesystem.rs
diff --git a/tiledb-sys/src/filter.rs → tiledb/sys/src/filter.rs b/tiledb-sys/src/filter.rs → tiledb/sys/src/filter.rs
diff --git a/tiledb-sys/src/filter_list.rs → tiledb/sys/src/filter_list.rs b/tiledb-sys/src/filter_list.rs → tiledb/sys/src/filter_list.rs
diff --git a/tiledb-sys/src/filter_option.rs → tiledb/sys/src/filter_option.rs b/tiledb-sys/src/filter_option.rs → tiledb/sys/src/filter_option.rs
diff --git a/tiledb-sys/src/filter_type.rs → tiledb/sys/src/filter_type.rs b/tiledb-sys/src/filter_type.rs → tiledb/sys/src/filter_type.rs
diff --git a/tiledb-sys/src/lib.rs → tiledb/sys/src/lib.rs b/tiledb-sys/src/lib.rs → tiledb/sys/src/lib.rs
diff --git a/tiledb-sys/src/object.rs → tiledb/sys/src/object.rs b/tiledb-sys/src/object.rs → tiledb/sys/src/object.rs
diff --git a/tiledb-sys/src/query.rs → tiledb/sys/src/query.rs b/tiledb-sys/src/query.rs → tiledb/sys/src/query.rs
diff --git a/tiledb-sys/src/schema.rs → tiledb/sys/src/schema.rs b/tiledb-sys/src/schema.rs → tiledb/sys/src/schema.rs
diff --git a/tiledb-sys/src/string.rs → tiledb/sys/src/string.rs b/tiledb-sys/src/string.rs → tiledb/sys/src/string.rs
diff --git a/tiledb-sys/src/subarray.rs → tiledb/sys/src/subarray.rs b/tiledb-sys/src/subarray.rs → tiledb/sys/src/subarray.rs
diff --git a/tiledb-sys/src/types.rs → tiledb/sys/src/types.rs b/tiledb-sys/src/types.rs → tiledb/sys/src/types.rs
diff --git a/tiledb-sys/wrapper.h → tiledb/sys/wrapper.h b/tiledb-sys/wrapper.h → tiledb/sys/wrapper.h