Scaffolding to support generating arbitrary filter pipelines (#23)

* Implement Filter with FilterData enum so we can derive PartialEq, Serialize, Deserialize

* Add proptest reflexive filter eq check

* Add test::filter::filter_list_arbitrary

* Datatype::is_real_type and is_byte_type

* Add FilterData::transform_datatype

* AttributeBuilder::filter_list consumes filter list

* Add FilterList::to_vec

* Arbitrary filter pipeline construction

tiledb/api/Cargo.toml

@@ -8,7 +8,7 @@ name = "tiledb"
 path = "src/lib.rs"
 
 [dependencies]
-serde = "1.0.136"
+serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.114"
 tiledb-sys = { workspace = true }
 

tiledb/api/src/array/attribute.rs

@@ -5,14 +5,13 @@ use std::fmt::{Debug, Formatter, Result as FmtResult};
 use std::ops::Deref;
 
 use serde_json::json;
-pub use tiledb_sys::Datatype;
 
 use crate::context::Context;
 use crate::convert::{BitsEq, CAPIConverter};
 use crate::error::Error;
 use crate::filter_list::{FilterList, RawFilterList};
 use crate::fn_typed;
-use crate::Result as TileDBResult;
+use crate::{Datatype, Result as TileDBResult};
 
 pub(crate) enum RawAttribute {
     Owned(*mut ffi::tiledb_attribute_t),
@@ -70,11 +69,7 @@ impl<'ctx> Attribute<'ctx> {
             ffi::tiledb_attribute_get_type(c_context, *self.raw, &mut c_dtype)
         };
         if res == ffi::TILEDB_OK {
-            if let Some(dtype) = Datatype::from_u32(c_dtype) {
-                Ok(dtype)
-            } else {
-                Err(Error::from("Invalid Datatype value returned by TileDB"))
-            }
+            Datatype::try_from(c_dtype)
         } else {
             Err(self.context.expect_last_error())
         }
@@ -238,13 +233,10 @@ impl<'ctx> Debug for Attribute<'ctx> {
                 } else {
                     None
                 },
-            /*
-            TODO
             "filters": match self.filter_list() {
                 Ok(fl) => format!("{:?}", fl),
                 Err(e) => format!("<error reading filters: {}>", e)
             },
-            */
             "raw": format!("{:p}", *self.raw)
         });
         write!(f, "{}", json)
@@ -274,15 +266,13 @@ impl<'c1, 'c2> PartialEq<Attribute<'c2>> for Attribute<'c1> {
             return false;
         }
 
-        /*
         let filter_match = match (self.filter_list(), other.filter_list()) {
-            (Ok(mine), Ok(theirs)) => unimplemented!(),
+            (Ok(mine), Ok(theirs)) => mine == theirs,
             _ => false,
         };
         if !filter_match {
             return false;
         }
-        */
 
         let cell_val_match = match (self.cell_val_num(), other.cell_val_num()) {
             (Ok(mine), Ok(theirs)) => mine == theirs,
@@ -545,12 +535,17 @@ mod test {
         {
             let flist2 = FilterListBuilder::new(&ctx)
                 .expect("Error creating filter list builder.")
-                .add_filter(NoopFilterBuilder::new(&ctx)?.build())?
-                .add_filter(BitWidthReductionFilterBuilder::new(&ctx)?.build())?
-                .add_filter(
-                    CompressionFilterBuilder::new(&ctx, CompressionType::Zstd)?
-                        .build(),
-                )?
+                .add_filter(Filter::create(&ctx, FilterData::None)?)?
+                .add_filter(Filter::create(
+                    &ctx,
+                    FilterData::BitWidthReduction { max_window: None },
+                )?)?
+                .add_filter(Filter::create(
+                    &ctx,
+                    FilterData::Compression(CompressionData::new(
+                        CompressionType::Zstd,
+                    )),
+                )?)?
                 .build();
 
             let attr = Builder::new(&ctx, "foo", Datatype::UInt8)

tiledb/api/src/array/dimension.rs

@@ -59,7 +59,7 @@ impl<'ctx> Dimension<'ctx> {
 
         assert_eq!(ffi::TILEDB_OK, c_ret);
 
-        Datatype::from_capi_enum(c_datatype)
+        Datatype::try_from(c_datatype).expect("Invalid dimension type")
     }
 
     pub fn domain<Conv: CAPIConverter>(&self) -> TileDBResult<[Conv; 2]> {
@@ -303,13 +303,12 @@ mod tests {
             let domain: [i32; 2] = [1, 4];
             let extent: i32 = 4;
             let fl = FilterListBuilder::new(&context)?
-                .add_filter(
-                    CompressionFilterBuilder::new(
-                        &context,
+                .add_filter(Filter::create(
+                    &context,
+                    FilterData::Compression(CompressionData::new(
                         CompressionType::Lz4,
-                    )?
-                    .build(),
-                )?
+                    )),
+                )?)?
                 .build();
             let dimension: Dimension = Builder::new::<i32>(
                 &context,
@@ -327,7 +326,13 @@ mod tests {
             assert_eq!(1, fl.get_num_filters().unwrap());
 
             let outlz4 = fl.get_filter(0).unwrap();
-            assert_eq!(ffi::FilterType::Lz4, outlz4.get_type().unwrap());
+            match outlz4.filter_data().expect("Error reading filter data") {
+                FilterData::Compression(CompressionData {
+                    kind: CompressionType::Lz4,
+                    ..
+                }) => (),
+                _ => unreachable!(),
+            }
         }
 
         Ok(())