From 0f1254d2b63c3499c8fa2916b12b8ad1fb89455f Mon Sep 17 00:00:00 2001 From: Joshua Mo Date: Tue, 4 Feb 2025 19:36:25 +0000 Subject: [PATCH 1/5] feat: fastembed integration --- Cargo.lock | 767 +++++++++++++++++++++++- Cargo.toml | 2 +- rig-fastembed/.gitignore | 1 + rig-fastembed/Cargo.toml | 20 + rig-fastembed/README.md | 4 + rig-fastembed/examples/rig-fastembed.rs | 81 +++ rig-fastembed/src/lib.rs | 154 +++++ 7 files changed, 1019 insertions(+), 10 deletions(-) create mode 100644 rig-fastembed/.gitignore create mode 100644 rig-fastembed/Cargo.toml create mode 100644 rig-fastembed/README.md create mode 100644 rig-fastembed/examples/rig-fastembed.rs create mode 100644 rig-fastembed/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e09baa4c..a584b1b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "allocator-api2" version = "0.2.20" @@ -94,12 +100,29 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + [[package]] name = "arc-swap" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "arrayvec" version = "0.7.6" @@ -663,6 +686,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "av1-grain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e335041290c43101ca215eed6f43ec437eb5a42125573f600fc3fa42b9bddd62" +dependencies = [ + "arrayvec", +] + [[package]] name = "aws-config" version = "1.5.10" @@ -1134,6 +1180,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -1158,6 +1210,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + [[package]] name = "bitvec" version = "1.0.1" @@ -1214,7 +1272,7 @@ dependencies = [ "hyperlocal", "log", "pin-project-lite", - "rustls 0.23.18", + "rustls 0.23.22", "rustls-native-certs 0.7.3", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -1283,6 +1341,12 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" + [[package]] name = "bumpalo" version = "3.16.0" @@ -1313,6 +1377,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.9.0" @@ -1415,6 +1485,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -1548,6 +1628,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "comfy-table" version = "7.1.3" @@ -1568,6 +1654,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", +] + [[package]] name = "const-hex" version = "1.14.0" @@ -2435,6 +2534,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2499,6 +2604,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etcetera" version = "0.8.0" @@ -2866,6 +2977,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "eyre" version = "0.6.12" @@ -2894,6 +3020,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" +[[package]] +name = "fastembed" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4b600d6b982794e86484cb5c5334ff6f2d4f9292b3e7ad92c46753840089b" +dependencies = [ + "anyhow", + "hf-hub", + "image", + "ndarray", + "ort", + "rayon", + "serde_json", + "tokenizers", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -2909,6 +3051,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.13.0" @@ -3232,6 +3383,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.31.1" @@ -3436,6 +3597,23 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-hub" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732" +dependencies = [ + "dirs", + "indicatif", + "log", + "native-tls", + "rand", + "serde", + "serde_json", + "thiserror 1.0.69", + "ureq", +] + [[package]] name = "hickory-proto" version = "0.24.1" @@ -3713,7 +3891,7 @@ dependencies = [ "http 1.1.0", "hyper 1.5.1", "hyper-util", - "rustls 0.23.18", + "rustls 0.23.22", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -3985,6 +4163,45 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "image" +version = "0.25.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", +] + +[[package]] +name = "imgref" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" + [[package]] name = "impl-codec" version = "0.6.0" @@ -4051,6 +4268,19 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.0", + "web-time", +] + [[package]] name = "indoc" version = "2.0.5" @@ -4078,6 +4308,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -4149,6 +4390,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.72" @@ -4669,6 +4916,12 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "levenshtein" version = "1.0.5" @@ -4751,6 +5004,16 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libm" version = "0.2.11" @@ -4822,6 +5085,15 @@ dependencies = [ "value-bag", ] +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lopdf" version = "0.34.0" @@ -4878,6 +5150,22 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + [[package]] name = "match_cfg" version = "0.1.0" @@ -4899,6 +5187,26 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "md-5" version = "0.10.6" @@ -4953,6 +5261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -5060,6 +5369,27 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "monostate" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "multimap" version = "0.10.0" @@ -5089,6 +5419,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "neo4rs" version = "0.8.0" @@ -5143,6 +5488,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -5209,6 +5560,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -5281,6 +5643,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.36.5" @@ -5333,6 +5701,28 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "open-fastrlp" version = "0.1.4" @@ -5417,6 +5807,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ort" +version = "2.0.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52afb44b6b0cffa9bf45e4d37e5a4935b0334a51570658e279e9e3e6cf324aa5" +dependencies = [ + "ndarray", + "ort-sys", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41d7757331aef2d04b9cb09b45583a59217628beaf91895b7e76187b6e8c088" +dependencies = [ + "flate2", + "pkg-config", + "sha2", + "tar", + "ureq", +] + [[package]] name = "outref" version = "0.5.1" @@ -5774,6 +6188,19 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "polling" version = "2.8.0" @@ -5805,6 +6232,21 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -5895,6 +6337,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +dependencies = [ + "quote", + "syn 2.0.89", +] + [[package]] name = "proptest" version = "1.6.0" @@ -6025,6 +6486,15 @@ dependencies = [ "tonic", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quanta" version = "0.11.1" @@ -6047,6 +6517,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.36.2" @@ -6068,7 +6544,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustls 0.23.22", "socket2 0.5.7", "thiserror 2.0.3", "tokio", @@ -6086,7 +6562,7 @@ dependencies = [ "rand", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustls 0.23.22", "rustls-pki-types", "slab", "thiserror 2.0.3", @@ -6179,6 +6655,56 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684" +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.12.1", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "system-deps", + "thiserror 1.0.69", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2413fd96bd0ea5cdeeb37eaf446a22e6ed7b981d792828721e74ded1980a45c6" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error 2.0.1", + "rav1e", + "rayon", + "rgb", +] + [[package]] name = "raw-cpuid" version = "10.7.0" @@ -6188,6 +6714,12 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.10.0" @@ -6198,6 +6730,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools 0.11.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.12.1" @@ -6358,7 +6901,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.18", + "rustls 0.23.22", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -6386,7 +6929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -6405,6 +6948,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" + [[package]] name = "rig-core" version = "0.7.0" @@ -6457,6 +7006,20 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rig-fastembed" +version = "0.1.0" +dependencies = [ + "anyhow", + "fastembed", + "rig-core", + "schemars", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "rig-lancedb" version = "0.2.3" @@ -6761,9 +7324,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" dependencies = [ "log", "once_cell", @@ -7287,6 +7850,21 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "similar" version = "2.6.0" @@ -7401,6 +7979,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "solang-parser" version = "0.3.3" @@ -7440,6 +8029,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlite-vec" version = "0.1.6" @@ -7855,6 +8456,19 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml", + "version-compare", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -8014,6 +8628,23 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.14.0" @@ -8144,6 +8775,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.36" @@ -8209,6 +8851,37 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" +dependencies = [ + "aho-corasick", + "derive_builder", + "esaxx-rs", + "getrandom", + "itertools 0.12.1", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax 0.8.5", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 1.0.69", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.43.0" @@ -8275,7 +8948,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.18", + "rustls 0.23.22", "rustls-pki-types", "tokio", ] @@ -8644,6 +9317,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.3" @@ -8674,6 +9356,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.7.1" @@ -8686,6 +9374,26 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "native-tls", + "once_cell", + "rustls 0.23.22", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "url", + "webpki-roots 0.26.7", +] + [[package]] name = "url" version = "2.5.4" @@ -8748,6 +9456,17 @@ dependencies = [ "serde", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.0" @@ -8766,6 +9485,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.5" @@ -9506,3 +10231,27 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index e1375fff..736e15e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,5 +9,5 @@ members = [ "rig-qdrant", "rig-core/rig-core-derive", "rig-sqlite", - "rig-eternalai", + "rig-eternalai", "rig-fastembed", ] diff --git a/rig-fastembed/.gitignore b/rig-fastembed/.gitignore new file mode 100644 index 00000000..971870eb --- /dev/null +++ b/rig-fastembed/.gitignore @@ -0,0 +1 @@ +.fastembed_cache diff --git a/rig-fastembed/Cargo.toml b/rig-fastembed/Cargo.toml new file mode 100644 index 00000000..faeea6a6 --- /dev/null +++ b/rig-fastembed/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rig-fastembed" +version = "0.1.0" +edition = "2021" + +[dependencies] +rig-core = { path = "../rig-core", version = "0.7.0" } +serde = { version = "1.0.193", features = ["derive"] } +serde_json = "1.0.108" +tracing = "0.1.40" +schemars = "0.8.16" +fastembed = "4.4.0" + +[dev-dependencies] +anyhow = "1.0.75" +tokio = { version = "1.34.0", features = ["full"] } + +[[example]] +name = "rig-fastembed" +required-features = ["rig-core/derive"] diff --git a/rig-fastembed/README.md b/rig-fastembed/README.md new file mode 100644 index 00000000..f7303a9a --- /dev/null +++ b/rig-fastembed/README.md @@ -0,0 +1,4 @@ +## Fastembed integration with Rig +This crate allows you to use [`fastembed-rs`](https://github.com/Anush008/fastembed-rs) with Rig. + +Unlike the providers found in the core crate, `fastembed` does not compile to the `wasm32-unknown-unknown` target. diff --git a/rig-fastembed/examples/rig-fastembed.rs b/rig-fastembed/examples/rig-fastembed.rs new file mode 100644 index 00000000..fa7a3e7e --- /dev/null +++ b/rig-fastembed/examples/rig-fastembed.rs @@ -0,0 +1,81 @@ +use rig::{ + embeddings::EmbeddingsBuilder, + vector_store::{in_memory_store::InMemoryVectorStore, VectorStoreIndex}, + Embed, +}; +use rig_fastembed::FastembedModel; +use serde::{Deserialize, Serialize}; + +// Shape of data that needs to be RAG'ed. +// The definition field will be used to generate embeddings. +#[derive(Embed, Clone, Deserialize, Debug, Serialize, Eq, PartialEq, Default)] +struct WordDefinition { + id: String, + word: String, + #[embed] + definitions: Vec, +} + +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + // Create OpenAI client + let fastembed_client = rig_fastembed::Client::new(); + + let embedding_model = fastembed_client.embedding_model(&FastembedModel::AllMiniLML6V2Q); + + let embeddings = EmbeddingsBuilder::new(embedding_model.clone()) + .documents(vec![ + WordDefinition { + id: "doc0".to_string(), + word: "flurbo".to_string(), + definitions: vec![ + "A green alien that lives on cold planets.".to_string(), + "A fictional digital currency that originated in the animated series Rick and Morty.".to_string() + ] + }, + WordDefinition { + id: "doc1".to_string(), + word: "glarb-glarb".to_string(), + definitions: vec![ + "An ancient tool used by the ancestors of the inhabitants of planet Jiro to farm the land.".to_string(), + "A fictional creature found in the distant, swampy marshlands of the planet Glibbo in the Andromeda galaxy.".to_string() + ] + }, + WordDefinition { + id: "doc2".to_string(), + word: "linglingdong".to_string(), + definitions: vec![ + "A term used by inhabitants of the sombrero galaxy to describe humans.".to_string(), + "A rare, mystical instrument crafted by the ancient monks of the Nebulon Mountain Ranges on the planet Quarm.".to_string() + ] + }, + ])? + .build() + .await?; + + // Create vector store with the embeddings + let vector_store = + InMemoryVectorStore::from_documents_with_id_f(embeddings, |doc| doc.id.clone()); + + // Create vector store index + let index = vector_store.index(embedding_model); + + let results = index + .top_n::("I need to buy something in a fictional universe. What type of money can I use for this?", 1) + .await? + .into_iter() + .map(|(score, id, doc)| (score, id, doc.word)) + .collect::>(); + + println!("Results: {:?}", results); + + let id_results = index + .top_n_ids("I need to buy something in a fictional universe. What type of money can I use for this?", 1) + .await? + .into_iter() + .collect::>(); + + println!("ID results: {:?}", id_results); + + Ok(()) +} diff --git a/rig-fastembed/src/lib.rs b/rig-fastembed/src/lib.rs new file mode 100644 index 00000000..49bb19e9 --- /dev/null +++ b/rig-fastembed/src/lib.rs @@ -0,0 +1,154 @@ +use std::sync::Arc; + +pub use fastembed::EmbeddingModel as FastembedModel; +use fastembed::{InitOptions, TextEmbedding}; +use rig::{ + embeddings::{self, EmbeddingError, EmbeddingsBuilder}, + Embed, +}; + +#[derive(Clone)] +pub struct Client; + +impl Default for Client { + fn default() -> Self { + Self::new() + } +} + +impl Client { + /// Create a new OpenAI client with the given API key. + pub fn new() -> Self { + Self + } + + /// Create an embedding model with the given name. + /// Note: default embedding dimension of 0 will be used if model is not known. + /// If this is the case, it's better to use function `embedding_model_with_ndims` + /// + /// # Example + /// ``` + /// use rig::providers::openai::{Client, self}; + /// + /// // Initialize the OpenAI client + /// let openai = Client::new("your-open-ai-api-key"); + /// + /// let embedding_model = openai.embedding_model(openai::TEXT_EMBEDDING_3_LARGE); + /// ``` + pub fn embedding_model(&self, model: &FastembedModel) -> EmbeddingModel { + let ndims = fetch_model_ndims(model); + + EmbeddingModel::new(model, ndims) + } + + /// Create an embedding builder with the given embedding model. + /// + /// # Example + /// ``` + /// use rig::providers::openai::{Client, self}; + /// + /// // Initialize the OpenAI client + /// let openai = Client::new("your-open-ai-api-key"); + /// + /// let embeddings = openai.embeddings(openai::TEXT_EMBEDDING_3_LARGE) + /// .simple_document("doc0", "Hello, world!") + /// .simple_document("doc1", "Goodbye, world!") + /// .build() + /// .await + /// .expect("Failed to embed documents"); + /// ``` + pub fn embeddings( + &self, + model: &fastembed::EmbeddingModel, + ) -> EmbeddingsBuilder { + EmbeddingsBuilder::new(self.embedding_model(model)) + } +} + +#[derive(Clone)] +pub struct EmbeddingModel { + embedder: Arc, + pub model: FastembedModel, + ndims: usize, +} + +impl EmbeddingModel { + pub fn new(model: &fastembed::EmbeddingModel, ndims: usize) -> Self { + let embedder = Arc::new( + TextEmbedding::try_new( + InitOptions::new(model.to_owned()).with_show_download_progress(true), + ) + .unwrap(), + ); + + Self { + embedder, + model: model.to_owned(), + ndims, + } + } +} + +impl embeddings::EmbeddingModel for EmbeddingModel { + const MAX_DOCUMENTS: usize = 1024; + + fn ndims(&self) -> usize { + self.ndims + } + + async fn embed_texts( + &self, + documents: impl IntoIterator, + ) -> Result, EmbeddingError> { + let documents_as_strings: Vec = documents.into_iter().collect(); + + let documents_as_vec = self + .embedder + .embed(documents_as_strings.clone(), None) + .map_err(|err| EmbeddingError::ProviderError(err.to_string()))?; + + let docs = documents_as_strings + .into_iter() + .zip(documents_as_vec) + .map(|(document, embedding)| embeddings::Embedding { + document, + vec: embedding.into_iter().map(|f| f as f64).collect(), + }) + .collect::>(); + + Ok(docs) + } +} + +/// As seen on the text embedding model cards file: +pub fn fetch_model_ndims(model: &FastembedModel) -> usize { + match model { + FastembedModel::AllMiniLML6V2 + | FastembedModel::AllMiniLML6V2Q + | FastembedModel::AllMiniLML12V2 + | FastembedModel::AllMiniLML12V2Q + | FastembedModel::BGESmallENV15 + | FastembedModel::BGESmallENV15Q + | FastembedModel::ParaphraseMLMiniLML12V2Q + | FastembedModel::ParaphraseMLMiniLML12V2 + | FastembedModel::MultilingualE5Small => 384, + FastembedModel::BGESmallZHV15 | FastembedModel::ClipVitB32 => 512, + FastembedModel::BGEBaseENV15 + | FastembedModel::BGEBaseENV15Q + | FastembedModel::NomicEmbedTextV1 + | FastembedModel::NomicEmbedTextV15 + | FastembedModel::NomicEmbedTextV15Q + | FastembedModel::ParaphraseMLMpnetBaseV2 + | FastembedModel::MultilingualE5Base + | FastembedModel::GTEBaseENV15 + | FastembedModel::GTEBaseENV15Q + | FastembedModel::JinaEmbeddingsV2BaseCode => 768, + FastembedModel::BGELargeENV15 + | FastembedModel::BGELargeENV15Q + | FastembedModel::MultilingualE5Large + | FastembedModel::MxbaiEmbedLargeV1 + | FastembedModel::MxbaiEmbedLargeV1Q + | FastembedModel::GTELargeENV15 + | FastembedModel::GTELargeENV15Q => 1024, + } +} From d72184651a1407b6512b9fdfef4961c76ce662b2 Mon Sep 17 00:00:00 2001 From: Joshua Mo Date: Tue, 4 Feb 2025 19:52:05 +0000 Subject: [PATCH 2/5] docs: ensure the docs are correct --- rig-fastembed/src/lib.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rig-fastembed/src/lib.rs b/rig-fastembed/src/lib.rs index 49bb19e9..94e5e002 100644 --- a/rig-fastembed/src/lib.rs +++ b/rig-fastembed/src/lib.rs @@ -17,7 +17,7 @@ impl Default for Client { } impl Client { - /// Create a new OpenAI client with the given API key. + /// Create a new Fastembed client. pub fn new() -> Self { Self } @@ -28,12 +28,12 @@ impl Client { /// /// # Example /// ``` - /// use rig::providers::openai::{Client, self}; + /// use rig_fastembed::{Client, FastembedModel}; /// /// // Initialize the OpenAI client - /// let openai = Client::new("your-open-ai-api-key"); + /// let fastembed_client = Client::new("your-open-ai-api-key"); /// - /// let embedding_model = openai.embedding_model(openai::TEXT_EMBEDDING_3_LARGE); + /// let embedding_model = fastembed_client.embedding_model(&FastembedModel::AllMiniLML6V2Q); /// ``` pub fn embedding_model(&self, model: &FastembedModel) -> EmbeddingModel { let ndims = fetch_model_ndims(model); @@ -45,12 +45,12 @@ impl Client { /// /// # Example /// ``` - /// use rig::providers::openai::{Client, self}; + /// use rig_fastembed::{Client, FastembedModel}; /// - /// // Initialize the OpenAI client - /// let openai = Client::new("your-open-ai-api-key"); + /// // Initialize the Fastembed client + /// let fastembed_client = Client::new(); /// - /// let embeddings = openai.embeddings(openai::TEXT_EMBEDDING_3_LARGE) + /// let embeddings = fastembed_client.embeddings(FastembedModel::AllMiniLML6V2Q) /// .simple_document("doc0", "Hello, world!") /// .simple_document("doc1", "Goodbye, world!") /// .build() From 5fba6ee879d0d99af7d02179dd54f3aedc559ab5 Mon Sep 17 00:00:00 2001 From: Joshua Mo Date: Wed, 5 Feb 2025 20:22:53 +0000 Subject: [PATCH 3/5] refactor: example name, add fastembed to readme --- README.md | 3 +++ rig-fastembed/examples/{rig-fastembed.rs => vector_search.rs} | 0 2 files changed, 3 insertions(+) rename rig-fastembed/examples/{rig-fastembed.rs => vector_search.rs} (100%) diff --git a/README.md b/README.md index d4f33794..f72e83d3 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,9 @@ Vector stores are available as separate companion-crates: - Qdrant vector store: [`rig-qdrant`](https://github.com/0xPlaygrounds/rig/tree/main/rig-qdrant) - SQLite vector store: [`rig-sqlite`](https://github.com/0xPlaygrounds/rig/tree/main/rig-sqlite) +The following providers are available as separate companion-crates: +- Fastembed: [`rig-fastembed`](https://github.com/0xPlaygrounds/rig/tree/main/rig-fastembed) +


diff --git a/rig-fastembed/examples/rig-fastembed.rs b/rig-fastembed/examples/vector_search.rs similarity index 100% rename from rig-fastembed/examples/rig-fastembed.rs rename to rig-fastembed/examples/vector_search.rs From 28475d0b4cd165da8ef73e6cc2d24dae24676287 Mon Sep 17 00:00:00 2001 From: Joshua Mo Date: Wed, 5 Feb 2025 20:24:26 +0000 Subject: [PATCH 4/5] chore: satisfy ci --- rig-fastembed/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rig-fastembed/Cargo.toml b/rig-fastembed/Cargo.toml index faeea6a6..e5810baa 100644 --- a/rig-fastembed/Cargo.toml +++ b/rig-fastembed/Cargo.toml @@ -16,5 +16,5 @@ anyhow = "1.0.75" tokio = { version = "1.34.0", features = ["full"] } [[example]] -name = "rig-fastembed" +name = "vector_search" required-features = ["rig-core/derive"] From ac44c932a34f7ab7d1336f78a75e965dff2435d2 Mon Sep 17 00:00:00 2001 From: Joshua Mo Date: Wed, 5 Feb 2025 20:39:07 +0000 Subject: [PATCH 5/5] chore: satisfy ci --- rig-core/examples/agent_evaluator_optimizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rig-core/examples/agent_evaluator_optimizer.rs b/rig-core/examples/agent_evaluator_optimizer.rs index afc6ae28..861705cb 100644 --- a/rig-core/examples/agent_evaluator_optimizer.rs +++ b/rig-core/examples/agent_evaluator_optimizer.rs @@ -79,7 +79,7 @@ async fn main() -> Result<(), anyhow::Error> { } else { let context = format!("{TASK}\n\n{}", eval_result.feedback); - response = generator_agent.prompt(&context).await.unwrap(); + response = generator_agent.prompt(context).await.unwrap(); memories.push(response.clone()); } }