From f8defdbce93ae1ab6aa1a9022d9a5fd182751b8f Mon Sep 17 00:00:00 2001 From: gaurshivangi Date: Tue, 28 May 2024 20:54:14 +0800 Subject: [PATCH 1/7] fc --- include/CANDY/ThresholdIndex.h | 235 ++++++++++++++++++++++++++++++ src/CANDY/CMakeLists.txt | 1 + src/CANDY/IndexTable.cpp | 5 + src/CANDY/ThresholdIndex.cpp | 168 +++++++++++++++++++++ test/CMakeLists.txt | 1 + test/SystemTest/ThresholdTest.cpp | 74 ++++++++++ 6 files changed, 484 insertions(+) create mode 100644 include/CANDY/ThresholdIndex.h create mode 100644 src/CANDY/ThresholdIndex.cpp create mode 100644 test/SystemTest/ThresholdTest.cpp diff --git a/include/CANDY/ThresholdIndex.h b/include/CANDY/ThresholdIndex.h new file mode 100644 index 000000000..0f0b7a640 --- /dev/null +++ b/include/CANDY/ThresholdIndex.h @@ -0,0 +1,235 @@ +/*! \file ThresholdIndex.h*/ +// +// Created by tony on 04/01/24. +// + +#ifndef CANDY_INCLUDE_CANDY_THRESHOLDINDEX_H_ +#define CANDY_INCLUDE_CANDY_THRESHOLDINDEX_H_ + +#include +#include +#include +#include +#include +#include +#include +namespace CANDY { + +/** + * @ingroup CANDY_lib_bottom The main body and interfaces of library function + * @{ + */ +/** + * @class ThresholdIndex CANDY/ThresholdIndex.h + * @brief The Threshold class of an index approach + */ +class ThresholdIndex { + protected: + faiss::MetricType faissMetric = faiss::METRIC_L2; + int64_t containerTier = 0; + int64_t dataThreshold; + std::string indexAlgorithm; + std::vector indices; + + + public: + bool isHPCStarted = false; + ThresholdIndex() { + + } + + ~ThresholdIndex() { + + } + /** + * @brief set the tier of this indexing, 0 refers the entry indexing + * @param tie the setting of tier number + * @note The parameter of tier idx affects nothing now, but will do something later + */ + virtual void setTier(int64_t tie) { + containerTier = tie; + } + /** + * @brief reset this index to inited status + */ + virtual void reset(); + /** + * @brief set the index-specific config related to one index + * @param cfg the config of this class, using raw class + * @note If there is any pre-built data structures, please load it in implementing this + * @note If there is any initial tensors to be stored, please load it after this by @ref loadInitialTensor + * @return bool whether the configuration is successful + */ + virtual bool setConfigClass(INTELLI::ConfigMap cfg); + /** + * @brief set the index-specfic config related to one index + * @param cfg the config of this class + * @note If there is any pre-built data structures, please load it in implementing this + * @note If there is any initial tensors to be stored, please load it after this by @ref loadInitialTensor + * @return bool whether the configuration is successful + */ + virtual bool setConfig(INTELLI::ConfigMapPtr cfg); + /** + * @brief some extra set-ups if the index has HPC fetures + * @return bool whether the HPC set-up is successful + */ + virtual bool startHPC(); + /** + * @brief insert a tensor + * @note This is majorly an online function + * @param t the tensor, some index need to be single row + * @return bool whether the insertion is successful + */ + virtual bool insertTensor(torch::Tensor &t); + + + virtual void createThresholdIndex(int64_t dimension); + + /** + * @brief load the initial tensors of a data base, use this BEFORE @ref insertTensor + * @note This is majorly an offline function, and may be different from @ref insertTensor for some indexes + * @param t the tensor, some index need to be single row + * @return bool whether the loading is successful + */ + virtual bool loadInitialTensor(torch::Tensor &t); + /** + * @brief delete a tensor, also online function + * @param t the tensor, some index needs to be single row + * @param k the number of nearest neighbors + * @return bool whether the deleting is successful + */ + virtual bool deleteTensor(torch::Tensor &t, int64_t k = 1); + + /** + * @brief revise a tensor + * @param t the tensor to be revised + * @param w the revised value + * @return bool whether the revising is successful + */ + virtual bool reviseTensor(torch::Tensor &t, torch::Tensor &w); + /** + * @brief search the k-NN of a query tensor, return their index + * @param t the tensor, allow multiple rows + * @param k the returned neighbors + * @return std::vector the index, follow faiss's order + */ + virtual std::vector searchIndex(torch::Tensor q, int64_t k); + + /** + * @brief return a vector of tensors according to some index + * @param idx the index, follow faiss's style, allow the KNN index of multiple queries + * @param k the returned neighbors, i.e., will be the number of rows of each returned tensor + * @return a vector of tensors, each tensor represent KNN results of one query in idx + */ + virtual std::vector getTensorByIndex(std::vector &idx, int64_t k); + /** + * @brief return the rawData of tensor + * @return The raw data stored in tensor + */ + virtual torch::Tensor rawData(); + /** + * @brief search the k-NN of a query tensor, return the result tensors + * @param t the tensor, allow multiple rows + * @param k the returned neighbors + * @return std::vector the result tensor for each row of query + */ + virtual std::vector searchTensor(torch::Tensor &q, int64_t k); + /** + * @brief some extra termination if the index has HPC features + * @return bool whether the HPC termination is successful + */ + virtual bool endHPC(); + /** + * @brief set the frozen level of online updating internal state + * @param frozenLv the level of frozen, 0 means freeze any online update in internal state + * @return whether the setting is successful + */ + virtual bool setFrozenLevel(int64_t frozenLv); + /** + * @brief offline build phase + * @param t the tensor for offline build + * @note This is to generate some offline data structures, NOT load offline tensors + * @note Please use @ref loadInitialTensor for loading initial tensors + * @return whether the building is successful + */ + virtual bool offlineBuild(torch::Tensor &t); + /** + * @brief a busy waiting for all pending operations to be done + * @return bool, whether the waiting is actually done; + */ + virtual bool waitPendingOperations(); + + /** + * @brief load the initial tensors of a data base along with its string objects, use this BEFORE @ref insertTensor + * @note This is majorly an offline function, and may be different from @ref insertTensor for some indexes + * @param t the tensor, some index need to be single row + * * @param strs the corresponding list of strings + * @return bool whether the loading is successful + */ + virtual bool loadInitialStringObject(torch::Tensor &t, std::vector &strs); + /** + * @brief insert a string object + * @note This is majorly an online function + * @param t the tensor, some index need to be single row + * @param strs the corresponding list of strings + * @return bool whether the insertion is successful + */ + virtual bool insertStringObject(torch::Tensor &t, std::vector &strs); + + /** + * @brief delete tensor along with its corresponding string object + * @note This is majorly an online function + * @param t the tensor, some index need to be single row + * @param k the number of nearest neighbors + * @return bool whether the delet is successful + */ + virtual bool deleteStringObject(torch::Tensor &t, int64_t k = 1); + + /** + * @brief search the k-NN of a query tensor, return the linked string objects + * @param t the tensor, allow multiple rows + * @param k the returned neighbors + * @return std::vector> the result object for each row of query + */ + virtual std::vector> searchStringObject(torch::Tensor &q, int64_t k); + /** + * @brief search the k-NN of a query tensor, return the linked string objects and original tensors + * @param t the tensor, allow multiple rows + * @param k the returned neighbors + * @return std::tuple,std::vector>> + */ + virtual std::tuple, std::vector>> searchTensorAndStringObject( + torch::Tensor &q, + int64_t k); + + /** + * @brief load the initial tensors and query distributions of a data base, use this BEFORE @ref insertTensor + * @note This is majorly an offline function, and may be different from @ref insertTensor for some indexes + * @param t the data tensor + * @param query the example query tensor + * @return bool whether the loading is successful + */ + virtual bool loadInitialTensorAndQueryDistribution(torch::Tensor &t, torch::Tensor &query); + + + }; + +/** + * @ingroup CANDY_lib_bottom + * @typedef ThresholdIndexPtr + * @brief The class to describe a shared pointer to @ref ThresholdIndex + + */ +typedef std::shared_ptr ThresholdIndexPtr; +/** + * @ingroup CANDY_lib_bottom + * @def newThresholdIndex + * @brief (Macro) To creat a new @ref ThresholdIndex shared pointer. + */ +#define newThresholdIndex std::make_shared +} +/** + * @} + */ + +#endif //INTELLISTREAM_INCLUDE_CPPALGOS_ThresholdCPPALGO_H_ diff --git a/src/CANDY/CMakeLists.txt b/src/CANDY/CMakeLists.txt index 8745ffe75..2f546765d 100644 --- a/src/CANDY/CMakeLists.txt +++ b/src/CANDY/CMakeLists.txt @@ -20,6 +20,7 @@ add_sources( NNDescentIndex.cpp FlannIndex.cpp DPGIndex.cpp + ThresholdIndex.cpp ) add_subdirectory(HashingModels) add_subdirectory(ParallelPartitionIndex) diff --git a/src/CANDY/IndexTable.cpp b/src/CANDY/IndexTable.cpp index 4b764aae1..2ae67e622 100644 --- a/src/CANDY/IndexTable.cpp +++ b/src/CANDY/IndexTable.cpp @@ -21,8 +21,11 @@ #include #include #include +#include #include #include + + #if CANDY_CL == 1 //#include #endif @@ -51,6 +54,8 @@ CANDY::IndexTable::IndexTable() { indexMap["nnDescent"] = newNNDescentIndex(); indexMap["Flann"] = newFlannIndex(); indexMap["DPG"] = newDPGIndex(); + //indexMap["threshold"] = newThresholdIndex(); + #if CANDY_CL == 1 // indexMap["cl"] = newCLMMCPPAlgo(); #endif diff --git a/src/CANDY/ThresholdIndex.cpp b/src/CANDY/ThresholdIndex.cpp new file mode 100644 index 000000000..1e4a1a015 --- /dev/null +++ b/src/CANDY/ThresholdIndex.cpp @@ -0,0 +1,168 @@ +/*! \file ThresholdIndex.cpp*/ +// +// Created by tony on 25/05/23. +// + +#include +#include +#include +#include +#include + + +void CANDY::ThresholdIndex::reset() { + +} + +bool CANDY::ThresholdIndex::offlineBuild(torch::Tensor &t) { + assert(t.size(1)); + auto index = new faiss::IndexFlatL2(t.size(1)); + //faiss::Index* newIndex = new faiss::IndexFlatL2(dimension); + //index->train(t.size(0), t.data_ptr()); + index->add(t.size(0), t.data_ptr()); + indices.push_back(index); + return true; +} + + +bool CANDY::ThresholdIndex::setConfig(INTELLI::ConfigMapPtr cfg) { + assert(cfg); + std::string metricType = cfg->tryString("metricType", "L2", true); + faissMetric = faiss::METRIC_L2; + if (metricType == "dot" || metricType == "IP" || metricType == "cossim") { + faissMetric = faiss::METRIC_INNER_PRODUCT; + } + + dataThreshold = cfg->tryI64("dataThreshold", 10000); + indexAlgorithm = cfg->tryString("indexAlgorithm", "HNSW"); + + + return false; +} +bool CANDY::ThresholdIndex::setConfigClass(INTELLI::ConfigMap cfg) { + INTELLI::ConfigMapPtr cfgPtr=newConfigMap(); + cfgPtr->loadFrom(cfg); + return setConfig(cfgPtr); +} +/* +bool CANDY::ThresholdIndex::setFrozenLevel(int64_t frozenLv) { + assert(frozenLv >= 0); + return false; +}*/ +bool CANDY::ThresholdIndex::insertTensor(torch::Tensor &t) { + assert(t.size(1)); + if (indices.empty() || indices.back()->ntotal >= dataThreshold) { + createThresholdIndex(t.size(1)); + } + indices.back()->add(t.size(0), t.data_ptr()); + return true; +} + +void CANDY::ThresholdIndex::createThresholdIndex(int64_t dimension) { + //auto index = new faiss::IndexFlatL2(dimension); + faiss::Index* index = new faiss::IndexFlatL2(dimension); + indices.push_back(index); +} + +/* +bool CANDY::ThresholdIndex::insertStringObject(torch::Tensor &t, std::vector &strs) { + assert(t.size(1)); + assert (strs.size()); + return false; +} +*/ +bool CANDY::ThresholdIndex::loadInitialTensor(torch::Tensor &t) { + return insertTensor(t); +} + +std::vector CANDY::ThresholdIndex::searchIndex(torch::Tensor q, int64_t k) { + assert(k > 0); + assert(q.size(1)); + std::vector ru(1); + return ru; +} + +std::vector CANDY::ThresholdIndex::searchTensor(torch::Tensor &q, int64_t k) { + assert(k > 0); + assert(q.size(1)); + + //auto idx = searchIndex(q, k); + std::vector allKnnIndices ; + + for (int64_t i = 0; i < indices.size(); ++i) { + std::vector knnIndices(k); + std::vector knnDistances(k); + indices[i]->search(q.size(0), q.data_ptr(), k, knnDistances.data(), knnIndices.data()); + for (faiss::idx_t index : knnIndices) { + allKnnIndices.push_back(torch::tensor(index)); + } + + //allKnnIndices.insert(allKnnIndices.end(), knnIndices.begin(), knnIndices.end()); + } + + return allKnnIndices; + //return getTensorByIndex(allKnnIndices, k); + +} +/* +torch::Tensor CANDY::ThresholdIndex::rawData() { + return torch::rand({1, 1}); +} + +bool CANDY::ThresholdIndex::startHPC() { + return false; +} + +bool CANDY::ThresholdIndex::endHPC() { + return false; +} +bool CANDY::ThresholdIndex::waitPendingOperations() { + return true; +} +std::tuple, std::vector>> CANDY::ThresholdIndex::searchTensorAndStringObject(torch::Tensor &q, int64_t k) { + auto ruT = searchTensor(q, k); + auto ruS = searchStringObject(q, k); + std::tuple, std::vector>> ru(ruT, ruS); + return ru; +} +bool CANDY::ThresholdIndex::loadInitialTensorAndQueryDistribution(torch::Tensor &t, torch::Tensor &query) { + assert(query.size(0) > 0); + return loadInitialTensor(t); +} + +std::vector> CANDY::ThresholdIndex::searchStringObject(torch::Tensor &q, int64_t k) { + assert(k > 0); + assert(q.size(1)); + std::vector> ru(1); + ru[0] = std::vector(1); + ru[0][0] = ""; + return ru; +} + +std::vector CANDY::ThresholdIndex::getTensorByIndex(std::vector &idx, int64_t k) { + assert(k > 0); + assert(idx.size()); + std::vector ru(1); + ru[0] = torch::rand({1, 1}); + return ru; +} + +bool CANDY::ThresholdIndex::loadInitialStringObject(torch::Tensor &t, std::vector &strs) { + return insertStringObject(t, strs); +} +bool CANDY::ThresholdIndex::deleteTensor(torch::Tensor &t, int64_t k) { + assert(t.size(1)); + assert(k > 0); + return false; +} + +bool CANDY::ThresholdIndex::deleteStringObject(torch::Tensor &t, int64_t k) { + assert(t.size(1)); + assert(k > 0); + return false; +} +bool CANDY::ThresholdIndex::reviseTensor(torch::Tensor &t, torch::Tensor &w) { + assert(t.size(1) == w.size(1)); + return false; +} +*/ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 62d447bb8..b37241a12 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -49,6 +49,7 @@ add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CAN add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDY) add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDY) add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDY) +add_catch_test(threshold_test SystemTest/ThresholdTest.cpp CANDY) if (ENABLE_OPENCL) add_catch_test(cl_test SystemTest/CLTest.cpp CANDY) endif () diff --git a/test/SystemTest/ThresholdTest.cpp b/test/SystemTest/ThresholdTest.cpp new file mode 100644 index 000000000..f881c30cc --- /dev/null +++ b/test/SystemTest/ThresholdTest.cpp @@ -0,0 +1,74 @@ +// +// Created by tony on 05/01/24. +// +#include + +#define CATCH_CONFIG_MAIN + +#include "catch.hpp" +#include +#include +using namespace std; +using namespace INTELLI; +using namespace torch; +using namespace CANDY; +TEST_CASE("Test threshold index", "[short]") +{ + int a = 0; + torch::manual_seed(114514); + // place your test here + INTELLI::ConfigMapPtr cfg = newConfigMap(); + CANDY::ThresholdIndex thresholdIndex; + //CANDY::IndexTable it; + //auto thresholdIndex = it.getIndex("threshold"); + cfg->edit("metricType", "L2"); + cfg->edit("dataThreshold", (int64_t) 100); + cfg->edit("indexAlgorithm", "HNSW"); + + thresholdIndex.setConfig(cfg); + + //offline + auto x_offline = torch::rand({150, 3}); + cout << "Offline build complete" << endl; + + //online + auto x_online = torch::rand({10, 3}); + REQUIRE(thresholdIndex.insertTensor(x_online) == true); + cout << "Online insertion complete" << endl; + + //search + size_t k = 3; + auto ru = thresholdIndex.searchTensor(x_online, k); + for (int64_t i = 0; i < x_online.size(0); i++) { + auto new_in = x_online.slice(0, i, i + 1); + cout << "Query tensor:" << new_in << endl; + cout << "Search result:" << ru[i] << endl << endl; + } + + /*CANDY::IndexTable it; + auto flatIdx = it.getIndex("flat"); + cfg->edit("vecDim", (int64_t) 4); + flatIdx->setConfig(cfg); + auto ta = torch::rand({1, 4}); + flatIdx->insertTensor(ta); + auto tb = torch::rand({1, 4}); + flatIdx->insertTensor(tb); + auto tc = torch::rand({1, 4}); + flatIdx->insertTensor(tc); + flatIdx->insertTensor(tb); + std::cout << "0.now, the data base is\n" << flatIdx->rawData() << std::endl; + auto ruIdx = flatIdx->searchIndex(tb, 2); + auto ruTensors = flatIdx->searchTensor(tb, 2); + std::cout << "1. now, do the query\n" << flatIdx->rawData() << std::endl; + for (uint64_t i = 0; i < ruIdx.size(); i++) { + std::cout << "result [" + to_string(i) + "]:idx=" << ruIdx[i] << std::endl; + } + std::cout << "get tensor\n" << ruTensors[0] << std::endl; + std::cout << "3. now, delete 2 similar\n" << std::endl; + flatIdx->deleteTensor(tb, 2); + std::cout << "the data base is\n" << flatIdx->rawData() << std::endl; + std::cout << "4. now, do the edit\n" << std::endl; + flatIdx->reviseTensor(tc, tb); + std::cout << "the data base is\n" << flatIdx->rawData() << std::endl; + REQUIRE(a == 0);*/ +} From b1f942f2a869017a777ea5c79e08488a55a81e2f Mon Sep 17 00:00:00 2001 From: tony <292224750@qq.com> Date: Wed, 29 May 2024 09:42:40 +0800 Subject: [PATCH 2/7] 1. fix readme and add one-key build script --- include/CANDY/ThresholdIndex.h | 56 ++++++++----------------------- src/CANDY/IndexTable.cpp | 2 +- src/CANDY/ThresholdIndex.cpp | 3 -- test/SystemTest/ThresholdTest.cpp | 3 +- 4 files changed, 17 insertions(+), 47 deletions(-) diff --git a/include/CANDY/ThresholdIndex.h b/include/CANDY/ThresholdIndex.h index 0f0b7a640..e8962a051 100644 --- a/include/CANDY/ThresholdIndex.h +++ b/include/CANDY/ThresholdIndex.h @@ -12,6 +12,7 @@ #include #include #include +#include #include namespace CANDY { @@ -23,7 +24,7 @@ namespace CANDY { * @class ThresholdIndex CANDY/ThresholdIndex.h * @brief The Threshold class of an index approach */ -class ThresholdIndex { +class ThresholdIndex: public AbstractIndex{ protected: faiss::MetricType faissMetric = faiss::METRIC_L2; int64_t containerTier = 0; @@ -73,7 +74,7 @@ class ThresholdIndex { * @brief some extra set-ups if the index has HPC fetures * @return bool whether the HPC set-up is successful */ - virtual bool startHPC(); + //virtual bool startHPC(); /** * @brief insert a tensor * @note This is majorly an online function @@ -91,14 +92,14 @@ class ThresholdIndex { * @param t the tensor, some index need to be single row * @return bool whether the loading is successful */ - virtual bool loadInitialTensor(torch::Tensor &t); + // virtual bool loadInitialTensor(torch::Tensor &t); /** * @brief delete a tensor, also online function * @param t the tensor, some index needs to be single row * @param k the number of nearest neighbors * @return bool whether the deleting is successful */ - virtual bool deleteTensor(torch::Tensor &t, int64_t k = 1); + //virtual bool deleteTensor(torch::Tensor &t, int64_t k = 1); /** * @brief revise a tensor @@ -106,7 +107,7 @@ class ThresholdIndex { * @param w the revised value * @return bool whether the revising is successful */ - virtual bool reviseTensor(torch::Tensor &t, torch::Tensor &w); + //virtual bool reviseTensor(torch::Tensor &t, torch::Tensor &w); /** * @brief search the k-NN of a query tensor, return their index * @param t the tensor, allow multiple rows @@ -115,19 +116,7 @@ class ThresholdIndex { */ virtual std::vector searchIndex(torch::Tensor q, int64_t k); - /** - * @brief return a vector of tensors according to some index - * @param idx the index, follow faiss's style, allow the KNN index of multiple queries - * @param k the returned neighbors, i.e., will be the number of rows of each returned tensor - * @return a vector of tensors, each tensor represent KNN results of one query in idx - */ - virtual std::vector getTensorByIndex(std::vector &idx, int64_t k); - /** - * @brief return the rawData of tensor - * @return The raw data stored in tensor - */ - virtual torch::Tensor rawData(); - /** +/** * @brief search the k-NN of a query tensor, return the result tensors * @param t the tensor, allow multiple rows * @param k the returned neighbors @@ -138,13 +127,13 @@ class ThresholdIndex { * @brief some extra termination if the index has HPC features * @return bool whether the HPC termination is successful */ - virtual bool endHPC(); + // virtual bool endHPC(); /** * @brief set the frozen level of online updating internal state * @param frozenLv the level of frozen, 0 means freeze any online update in internal state * @return whether the setting is successful */ - virtual bool setFrozenLevel(int64_t frozenLv); + // virtual bool setFrozenLevel(int64_t frozenLv); /** * @brief offline build phase * @param t the tensor for offline build @@ -157,7 +146,7 @@ class ThresholdIndex { * @brief a busy waiting for all pending operations to be done * @return bool, whether the waiting is actually done; */ - virtual bool waitPendingOperations(); + //virtual bool waitPendingOperations(); /** * @brief load the initial tensors of a data base along with its string objects, use this BEFORE @ref insertTensor @@ -166,7 +155,7 @@ class ThresholdIndex { * * @param strs the corresponding list of strings * @return bool whether the loading is successful */ - virtual bool loadInitialStringObject(torch::Tensor &t, std::vector &strs); + //virtual bool loadInitialStringObject(torch::Tensor &t, std::vector &strs); /** * @brief insert a string object * @note This is majorly an online function @@ -174,7 +163,7 @@ class ThresholdIndex { * @param strs the corresponding list of strings * @return bool whether the insertion is successful */ - virtual bool insertStringObject(torch::Tensor &t, std::vector &strs); + // virtual bool insertStringObject(torch::Tensor &t, std::vector &strs); /** * @brief delete tensor along with its corresponding string object @@ -183,7 +172,7 @@ class ThresholdIndex { * @param k the number of nearest neighbors * @return bool whether the delet is successful */ - virtual bool deleteStringObject(torch::Tensor &t, int64_t k = 1); + // virtual bool deleteStringObject(torch::Tensor &t, int64_t k = 1); /** * @brief search the k-NN of a query tensor, return the linked string objects @@ -191,25 +180,8 @@ class ThresholdIndex { * @param k the returned neighbors * @return std::vector> the result object for each row of query */ - virtual std::vector> searchStringObject(torch::Tensor &q, int64_t k); - /** - * @brief search the k-NN of a query tensor, return the linked string objects and original tensors - * @param t the tensor, allow multiple rows - * @param k the returned neighbors - * @return std::tuple,std::vector>> - */ - virtual std::tuple, std::vector>> searchTensorAndStringObject( - torch::Tensor &q, - int64_t k); + //virtual std::vector> searchStringObject(torch::Tensor &q, int64_t k); - /** - * @brief load the initial tensors and query distributions of a data base, use this BEFORE @ref insertTensor - * @note This is majorly an offline function, and may be different from @ref insertTensor for some indexes - * @param t the data tensor - * @param query the example query tensor - * @return bool whether the loading is successful - */ - virtual bool loadInitialTensorAndQueryDistribution(torch::Tensor &t, torch::Tensor &query); }; diff --git a/src/CANDY/IndexTable.cpp b/src/CANDY/IndexTable.cpp index 2ae67e622..d9be0d466 100644 --- a/src/CANDY/IndexTable.cpp +++ b/src/CANDY/IndexTable.cpp @@ -54,7 +54,7 @@ CANDY::IndexTable::IndexTable() { indexMap["nnDescent"] = newNNDescentIndex(); indexMap["Flann"] = newFlannIndex(); indexMap["DPG"] = newDPGIndex(); - //indexMap["threshold"] = newThresholdIndex(); + indexMap["threshold"] = newThresholdIndex(); #if CANDY_CL == 1 // indexMap["cl"] = newCLMMCPPAlgo(); diff --git a/src/CANDY/ThresholdIndex.cpp b/src/CANDY/ThresholdIndex.cpp index 1e4a1a015..863469e45 100644 --- a/src/CANDY/ThresholdIndex.cpp +++ b/src/CANDY/ThresholdIndex.cpp @@ -71,9 +71,6 @@ bool CANDY::ThresholdIndex::insertStringObject(torch::Tensor &t, std::vector CANDY::ThresholdIndex::searchIndex(torch::Tensor q, int64_t k) { assert(k > 0); diff --git a/test/SystemTest/ThresholdTest.cpp b/test/SystemTest/ThresholdTest.cpp index f881c30cc..2a6cd7417 100644 --- a/test/SystemTest/ThresholdTest.cpp +++ b/test/SystemTest/ThresholdTest.cpp @@ -8,13 +8,14 @@ #include "catch.hpp" #include #include +#include using namespace std; using namespace INTELLI; using namespace torch; using namespace CANDY; TEST_CASE("Test threshold index", "[short]") { - int a = 0; + //int a = 0; torch::manual_seed(114514); // place your test here INTELLI::ConfigMapPtr cfg = newConfigMap(); From 04c5c758bba164ef365e13022241c88a598804da Mon Sep 17 00:00:00 2001 From: gaurshivangi Date: Thu, 30 May 2024 15:37:07 +0800 Subject: [PATCH 3/7] header file changed --- include/CANDY/ThresholdIndex.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/CANDY/ThresholdIndex.h b/include/CANDY/ThresholdIndex.h index e8962a051..5c3279cc8 100644 --- a/include/CANDY/ThresholdIndex.h +++ b/include/CANDY/ThresholdIndex.h @@ -47,9 +47,9 @@ class ThresholdIndex: public AbstractIndex{ * @param tie the setting of tier number * @note The parameter of tier idx affects nothing now, but will do something later */ - virtual void setTier(int64_t tie) { + /*virtual void setTier(int64_t tie) { containerTier = tie; - } + }*/ /** * @brief reset this index to inited status */ From b016dbc66d1d2b2033486a3c860127fb2e228671 Mon Sep 17 00:00:00 2001 From: gaurshivangi Date: Fri, 31 May 2024 19:00:21 +0800 Subject: [PATCH 4/7] test corrected --- src/CANDY/ThresholdIndex.cpp | 58 +++++++++++++++++++++---------- test/SystemTest/ThresholdTest.cpp | 54 +++++++++++++++++++++------- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/CANDY/ThresholdIndex.cpp b/src/CANDY/ThresholdIndex.cpp index 863469e45..5093709b0 100644 --- a/src/CANDY/ThresholdIndex.cpp +++ b/src/CANDY/ThresholdIndex.cpp @@ -16,11 +16,15 @@ void CANDY::ThresholdIndex::reset() { bool CANDY::ThresholdIndex::offlineBuild(torch::Tensor &t) { assert(t.size(1)); - auto index = new faiss::IndexFlatL2(t.size(1)); - //faiss::Index* newIndex = new faiss::IndexFlatL2(dimension); - //index->train(t.size(0), t.data_ptr()); - index->add(t.size(0), t.data_ptr()); - indices.push_back(index); + if (indices.empty() || indices.back()->ntotal >= dataThreshold) { + createThresholdIndex(t.size(1)); + } + //auto index = new faiss::IndexFlatL2(t.size(1)); + //index->add(t.size(0), t.data_ptr()); + //indices.push_back(index); + + indices.back()->add(t.size(0), t.data_ptr()); + return true; } @@ -39,6 +43,7 @@ bool CANDY::ThresholdIndex::setConfig(INTELLI::ConfigMapPtr cfg) { return false; } + bool CANDY::ThresholdIndex::setConfigClass(INTELLI::ConfigMap cfg) { INTELLI::ConfigMapPtr cfgPtr=newConfigMap(); cfgPtr->loadFrom(cfg); @@ -60,7 +65,8 @@ bool CANDY::ThresholdIndex::insertTensor(torch::Tensor &t) { void CANDY::ThresholdIndex::createThresholdIndex(int64_t dimension) { //auto index = new faiss::IndexFlatL2(dimension); - faiss::Index* index = new faiss::IndexFlatL2(dimension); + //int M=32; + auto index = new faiss::IndexFlat(dimension); indices.push_back(index); } @@ -84,23 +90,39 @@ std::vector CANDY::ThresholdIndex::searchTensor(torch::Tensor &q, assert(q.size(1)); //auto idx = searchIndex(q, k); - std::vector allKnnIndices ; + std::vector Indicesx ; + std::vector Distances; for (int64_t i = 0; i < indices.size(); ++i) { - std::vector knnIndices(k); - std::vector knnDistances(k); - indices[i]->search(q.size(0), q.data_ptr(), k, knnDistances.data(), knnIndices.data()); - for (faiss::idx_t index : knnIndices) { - allKnnIndices.push_back(torch::tensor(index)); - } - - //allKnnIndices.insert(allKnnIndices.end(), knnIndices.begin(), knnIndices.end()); + int64_t querySize = q.size(0); + std::vector inx(k * querySize); + std::vector dist(k * querySize); + indices[i]->search(q.size(0), q.data_ptr(), k, dist.data(), inx.data()); + //for (faiss::idx_t index : inx) { + // inx.push_back(torch::tensor(index)); + //} + Indicesx.insert(Indicesx.end(), inx.begin(), inx.end()); + Distances.insert(Distances.end(), dist.begin(), dist.end()); + } - return allKnnIndices; - //return getTensorByIndex(allKnnIndices, k); + std::vector> knn; + for (size_t i = 0; i < Indicesx.size(); ++i) { + knn.emplace_back(Distances[i], Indicesx[i]); + } + + std::sort(knn.begin(), knn.end(), [](const auto &a, const auto &b) { + return a.first < b.first; + }); + + std::vector topK; + for (int64_t i = 0; i < k && i < knn.size(); ++i) { + topK.push_back(torch::tensor(knn[i].second)); + } + + return topK; +} -} /* torch::Tensor CANDY::ThresholdIndex::rawData() { return torch::rand({1, 1}); diff --git a/test/SystemTest/ThresholdTest.cpp b/test/SystemTest/ThresholdTest.cpp index 2a6cd7417..7a131d17d 100644 --- a/test/SystemTest/ThresholdTest.cpp +++ b/test/SystemTest/ThresholdTest.cpp @@ -23,28 +23,53 @@ TEST_CASE("Test threshold index", "[short]") //CANDY::IndexTable it; //auto thresholdIndex = it.getIndex("threshold"); cfg->edit("metricType", "L2"); - cfg->edit("dataThreshold", (int64_t) 100); + cfg->edit("dataThreshold", (int64_t) 8); cfg->edit("indexAlgorithm", "HNSW"); thresholdIndex.setConfig(cfg); - + /* //offline auto x_offline = torch::rand({150, 3}); + REQUIRE(thresholdIndex.offlineBuild(x_offline) == true); cout << "Offline build complete" << endl; - //online + //online --> need change not correct auto x_online = torch::rand({10, 3}); REQUIRE(thresholdIndex.insertTensor(x_online) == true); cout << "Online insertion complete" << endl; - + //search size_t k = 3; - auto ru = thresholdIndex.searchTensor(x_online, k); - for (int64_t i = 0; i < x_online.size(0); i++) { - auto new_in = x_online.slice(0, i, i + 1); - cout << "Query tensor:" << new_in << endl; - cout << "Search result:" << ru[i] << endl << endl; - } + auto tb = torch::rand({1, 4}); + auto ruTensors = thresholdIndex.searchTensor(tb, 2); + std::cout << "get tensor\n" << ruTensors[0] << std::endl; + */ + + auto db = torch::rand({6, 4}); + thresholdIndex.insertTensor(db); + std::cout << "data base is\n" << db << std::endl; + auto query = db.slice(0, 2, 3); + std::cout << "query is\n" << query << std::endl; + auto thRu = thresholdIndex.searchTensor(query, 2); + std::cout << "get tensor\n" << thRu[0] << std::endl; + + auto ta = torch::rand({1, 4}); + thresholdIndex.insertTensor(ta); + auto tb = torch::rand({1, 4}); + thresholdIndex.insertTensor(tb); + auto tc = torch::rand({1, 4}); + thresholdIndex.insertTensor(tc); + thresholdIndex.insertTensor(tb); + thresholdIndex.insertTensor(ta); + std::cout << "0.now, the data base is\n" << thresholdIndex.rawData() << std::endl; + auto ruTensors = thresholdIndex.searchTensor(tb, 2); + std::cout << "1. now, do the query\n" << thresholdIndex.rawData() << std::endl; + + std::cout << "get tensor\n" << ruTensors[0] << std::endl; + + + + } /*CANDY::IndexTable it; auto flatIdx = it.getIndex("flat"); @@ -71,5 +96,10 @@ TEST_CASE("Test threshold index", "[short]") std::cout << "4. now, do the edit\n" << std::endl; flatIdx->reviseTensor(tc, tb); std::cout << "the data base is\n" << flatIdx->rawData() << std::endl; - REQUIRE(a == 0);*/ -} + REQUIRE(a == 0); + auto ru = thresholdIndex.searchTensor(x_online, k); + for (int64_t i = 0; i < x_online.size(0); i++) { + auto new_in = x_online.slice(0, i, i + 1); + cout << "Query tensor:" << new_in << endl; + cout << "Search result:" << ru[i] << endl << endl;*/ + From 2f6dd1fd073bfd285bf4b9e9423bfbf078007fea Mon Sep 17 00:00:00 2001 From: gaurshivangi Date: Tue, 4 Jun 2024 20:23:31 +0800 Subject: [PATCH 5/7] different index selection added --- include/CANDY/ThresholdIndex.h | 15 ++-- src/CANDY/ThresholdIndex.cpp | 104 +++++++++++++++++++-------- test/SystemTest/ThresholdTest.cpp | 115 ++++++++++++++++++++---------- 3 files changed, 165 insertions(+), 69 deletions(-) diff --git a/include/CANDY/ThresholdIndex.h b/include/CANDY/ThresholdIndex.h index 5c3279cc8..b8c24d15b 100644 --- a/include/CANDY/ThresholdIndex.h +++ b/include/CANDY/ThresholdIndex.h @@ -29,8 +29,10 @@ class ThresholdIndex: public AbstractIndex{ faiss::MetricType faissMetric = faiss::METRIC_L2; int64_t containerTier = 0; int64_t dataThreshold; + int64_t dataVolume; std::string indexAlgorithm; - std::vector indices; + //std::vector indices; + std::vector indices; public: @@ -81,10 +83,13 @@ class ThresholdIndex: public AbstractIndex{ * @param t the tensor, some index need to be single row * @return bool whether the insertion is successful */ - virtual bool insertTensor(torch::Tensor &t); + virtual bool insertTensor_th(torch::Tensor &t, std::string nameTag); - virtual void createThresholdIndex(int64_t dimension); + virtual void createThresholdIndex(int64_t dimension, std::string nameTag); + //CANDY::AbstractIndexPtr createIndex(const std::string& nameTag); + + //CANDY::AbstractIndexPtr createIndex(std::string nameTag); /** * @brief load the initial tensors of a data base, use this BEFORE @ref insertTensor @@ -122,7 +127,7 @@ class ThresholdIndex: public AbstractIndex{ * @param k the returned neighbors * @return std::vector the result tensor for each row of query */ - virtual std::vector searchTensor(torch::Tensor &q, int64_t k); + virtual std::vector searchTensor_th(torch::Tensor &q, int64_t k); /** * @brief some extra termination if the index has HPC features * @return bool whether the HPC termination is successful @@ -141,7 +146,7 @@ class ThresholdIndex: public AbstractIndex{ * @note Please use @ref loadInitialTensor for loading initial tensors * @return whether the building is successful */ - virtual bool offlineBuild(torch::Tensor &t); + virtual bool offlineBuild(torch::Tensor &t, std::string nameTag); /** * @brief a busy waiting for all pending operations to be done * @return bool, whether the waiting is actually done; diff --git a/src/CANDY/ThresholdIndex.cpp b/src/CANDY/ThresholdIndex.cpp index 5093709b0..b74d63f39 100644 --- a/src/CANDY/ThresholdIndex.cpp +++ b/src/CANDY/ThresholdIndex.cpp @@ -8,22 +8,25 @@ #include #include #include +#include + void CANDY::ThresholdIndex::reset() { } -bool CANDY::ThresholdIndex::offlineBuild(torch::Tensor &t) { +bool CANDY::ThresholdIndex::offlineBuild(torch::Tensor &t, std::string nameTag) { assert(t.size(1)); - if (indices.empty() || indices.back()->ntotal >= dataThreshold) { - createThresholdIndex(t.size(1)); + if (indices.empty() || dataVolume >= dataThreshold) { + createThresholdIndex(t.size(1), nameTag); } //auto index = new faiss::IndexFlatL2(t.size(1)); //index->add(t.size(0), t.data_ptr()); //indices.push_back(index); - indices.back()->add(t.size(0), t.data_ptr()); + indices.back()->insertTensor(t); + dataVolume++; return true; } @@ -37,7 +40,7 @@ bool CANDY::ThresholdIndex::setConfig(INTELLI::ConfigMapPtr cfg) { faissMetric = faiss::METRIC_INNER_PRODUCT; } - dataThreshold = cfg->tryI64("dataThreshold", 10000); + dataThreshold = cfg->tryI64("dataThreshold", 100); indexAlgorithm = cfg->tryString("indexAlgorithm", "HNSW"); @@ -54,22 +57,45 @@ bool CANDY::ThresholdIndex::setFrozenLevel(int64_t frozenLv) { assert(frozenLv >= 0); return false; }*/ -bool CANDY::ThresholdIndex::insertTensor(torch::Tensor &t) { +bool CANDY::ThresholdIndex::insertTensor_th(torch::Tensor &t ,std::string nameTag) { assert(t.size(1)); - if (indices.empty() || indices.back()->ntotal >= dataThreshold) { - createThresholdIndex(t.size(1)); + if (indices.empty() || dataVolume >= dataThreshold) { + createThresholdIndex(t.size(1), nameTag); } - indices.back()->add(t.size(0), t.data_ptr()); + indices.back()->insertTensor(t); + + dataVolume++; return true; } -void CANDY::ThresholdIndex::createThresholdIndex(int64_t dimension) { +void CANDY::ThresholdIndex::createThresholdIndex(int64_t dimension, std::string nameTag) { //auto index = new faiss::IndexFlatL2(dimension); //int M=32; - auto index = new faiss::IndexFlat(dimension); - indices.push_back(index); -} + //auto index = AbstractIndexPtr::createIndex(nameTag); + + IndexTable tab; + auto ru= tab.getIndex(nameTag); + + if(ru==nullptr){ + INTELLI_ERROR("No index named "+nameTag+", return flat"); + nameTag="flat"; + INTELLI::ConfigMapPtr cfg_new = newConfigMap(); + cfg_new->edit("vecDim", (int64_t) 3); + cfg_new->edit("M", (int64_t) 4); + ru->setConfig(cfg_new); + indices.push_back(tab.getIndex(nameTag)); + dataVolume=0; + return; + } + INTELLI::ConfigMapPtr cfg_new = newConfigMap(); + cfg_new->edit("vecDim", (int64_t) 3); + cfg_new->edit("M", (int64_t) 4); + ru->setConfig(cfg_new); + dataVolume=0; + indices.push_back(ru); + return; +} /* bool CANDY::ThresholdIndex::insertStringObject(torch::Tensor &t, std::vector &strs) { assert(t.size(1)); @@ -85,28 +111,51 @@ std::vector CANDY::ThresholdIndex::searchIndex(torch::Tensor q, in return ru; } -std::vector CANDY::ThresholdIndex::searchTensor(torch::Tensor &q, int64_t k) { +float getDist(const torch::Tensor& a, const torch::Tensor& b) { + return torch::norm(a - b).item(); +} + +std::vector CANDY::ThresholdIndex::searchTensor_th(torch::Tensor &q, int64_t k) { assert(k > 0); assert(q.size(1)); //auto idx = searchIndex(q, k); - std::vector Indicesx ; - std::vector Distances; + std::vector Indicesx ; for (int64_t i = 0; i < indices.size(); ++i) { int64_t querySize = q.size(0); - std::vector inx(k * querySize); - std::vector dist(k * querySize); - indices[i]->search(q.size(0), q.data_ptr(), k, dist.data(), inx.data()); - //for (faiss::idx_t index : inx) { - // inx.push_back(torch::tensor(index)); - //} - Indicesx.insert(Indicesx.end(), inx.begin(), inx.end()); - Distances.insert(Distances.end(), dist.begin(), dist.end()); + auto inx = indices[i]->searchTensor(q,k); + //std::cout << "Index " << i << " returned " << inx.size() << " results" << endl; + for(int64_t j=0; j< k; j++) + {//std::cout << "Result tensor from index " << i << ": " << inx[j] << std::endl; + Indicesx.push_back(inx[0].slice(0, j, j + 1) ); + } + //Indicesx.insert(Indicesx.end(), inx.begin(), inx.end()) +} + + if(Indicesx.size()>k) + { + std::vector> dist; + for (int64_t i = 0; i < Indicesx.size(); ++i) { + float distance = getDist(q, Indicesx[i]); + dist.push_back(std::make_pair(distance, i)); + } + + std::sort(dist.begin(), dist.end()); + std::vector topK; + + for (int64_t i = 0; i < k; ++i) { + topK.push_back(Indicesx[dist[i].second]); + } + return topK; } - std::vector> knn; + return Indicesx; + +} + + /*std::vector> knn; for (size_t i = 0; i < Indicesx.size(); ++i) { knn.emplace_back(Distances[i], Indicesx[i]); } @@ -119,9 +168,8 @@ std::vector CANDY::ThresholdIndex::searchTensor(torch::Tensor &q, for (int64_t i = 0; i < k && i < knn.size(); ++i) { topK.push_back(torch::tensor(knn[i].second)); } - - return topK; -} + */ + /* torch::Tensor CANDY::ThresholdIndex::rawData() { diff --git a/test/SystemTest/ThresholdTest.cpp b/test/SystemTest/ThresholdTest.cpp index 7a131d17d..60d308055 100644 --- a/test/SystemTest/ThresholdTest.cpp +++ b/test/SystemTest/ThresholdTest.cpp @@ -9,25 +9,86 @@ #include #include #include +#include +#include using namespace std; using namespace INTELLI; using namespace torch; using namespace CANDY; -TEST_CASE("Test threshold index", "[short]") -{ - //int a = 0; - torch::manual_seed(114514); - // place your test here - INTELLI::ConfigMapPtr cfg = newConfigMap(); - CANDY::ThresholdIndex thresholdIndex; - //CANDY::IndexTable it; - //auto thresholdIndex = it.getIndex("threshold"); +TEST_CASE("Test threshold index", "[short]") { + torch::manual_seed(114514); + INTELLI::ConfigMapPtr cfg = newConfigMap(); + CANDY::ThresholdIndex thresholdIndex; + cfg->edit("metricType", "L2"); - cfg->edit("dataThreshold", (int64_t) 8); - cfg->edit("indexAlgorithm", "HNSW"); + cfg->edit("dataThreshold", (int64_t) 60); thresholdIndex.setConfig(cfg); + + auto ta = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(ta, "HNSWNaive"); + auto tb = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(tb, "HNSWNaive"); + auto tc = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(tc, "HNSWNaive"); + + for(int i = 0; i < 100; i++) { + auto x_in = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(x_in, "HNSWNaive"); + } + + thresholdIndex.insertTensor_th(ta, "HNSWNaive"); + thresholdIndex.insertTensor_th(tb, "HNSWNaive"); + thresholdIndex.insertTensor_th(tb, "HNSWNaive"); + + std::cout << "Insertion finished" << std::endl; + + // Search for 'ta' + std::cout << "1. Now, do the query\n" << ta << std::endl; + auto ruTensors = thresholdIndex.searchTensor_th(ta, 2); + std::cout << "Get tensor\n"; + + for (const auto& tensor : ruTensors) { + std::cout << tensor << std::endl; + } + std::cout << "Total results: " << ruTensors.size() << std::endl; +} + + //size_t k = 1; + /* + for(int i=0; i<20; i++) + { + auto x_in = torch::rand({1, 3}); + hnswIdx.insertTensor(x_in); + } + cout << "insertion finish" << endl; + auto q = torch::rand({1, 3}); + std::cout << "1. now, do the query\n" << q << std::endl; + auto ruTensors = hnswIdx.searchTensor(q, 3); + std::cout << "get tensor\n" << ruTensors[0] << std::endl; + auto y_in = torch::rand({110, 3}); + thresholdIndex.insertTensor_th(y_in, "HNSWNaive"); + auto ru = thresholdIndex.searchTensor_th(x_in, k); + for (int64_t i = 0; i < x_in.size(0); i++) { + + auto new_in = newTensor(x_in.slice(0, i, i + 1)); + cout << "looking for" << *new_in << endl; + cout << endl << ru[i] << endl << endl; + }*/ /* + auto ta = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(ta, "HNSWNaive"); + auto tb = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(tb, "HNSWNaive"); + auto tc = torch::rand({1, 3}); + thresholdIndex.insertTensor_th(ta, "HNSWNaive"); + thresholdIndex.insertTensor_th(tb, "HNSWNaive"); + thresholdIndex.insertTensor_th(tc, "HNSWNaive"); + //std::cout << "0.now, the data base is\n" << thresholdIndex.rawData() << std::endl; + */ + + + /* //offline auto x_offline = torch::rand({150, 3}); REQUIRE(thresholdIndex.offlineBuild(x_offline) == true); @@ -45,31 +106,13 @@ TEST_CASE("Test threshold index", "[short]") std::cout << "get tensor\n" << ruTensors[0] << std::endl; */ - auto db = torch::rand({6, 4}); - thresholdIndex.insertTensor(db); - std::cout << "data base is\n" << db << std::endl; - auto query = db.slice(0, 2, 3); - std::cout << "query is\n" << query << std::endl; - auto thRu = thresholdIndex.searchTensor(query, 2); - std::cout << "get tensor\n" << thRu[0] << std::endl; - - auto ta = torch::rand({1, 4}); - thresholdIndex.insertTensor(ta); - auto tb = torch::rand({1, 4}); - thresholdIndex.insertTensor(tb); - auto tc = torch::rand({1, 4}); - thresholdIndex.insertTensor(tc); - thresholdIndex.insertTensor(tb); - thresholdIndex.insertTensor(ta); - std::cout << "0.now, the data base is\n" << thresholdIndex.rawData() << std::endl; - auto ruTensors = thresholdIndex.searchTensor(tb, 2); - std::cout << "1. now, do the query\n" << thresholdIndex.rawData() << std::endl; - - std::cout << "get tensor\n" << ruTensors[0] << std::endl; - - - - } + //auto db = torch::rand({6, 4}); + //thresholdIndex.insertTensor_th(db, "HNSWNaive"); + //std::cout << "data base is\n" << db << std::endl; + //auto query = db.slice(0, 2, 3); + //std::cout << "query is\n" << query << std::endl; + //auto thRu = thresholdIndex.searchTensor(query, 2); + //std::cout << "get tensor\n" << thRu[0] << std::endl; /*CANDY::IndexTable it; auto flatIdx = it.getIndex("flat"); From 88ede5f979157623eb25db450324e2fa7855a421 Mon Sep 17 00:00:00 2001 From: Regis Wu <73592273+Rubatozh@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:33:24 +0800 Subject: [PATCH 6/7] Update test/CMakeLists.txt --- test/CMakeLists.txt | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a79e62e07..2f198acb6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -36,21 +36,7 @@ macro(add_catch_test_ray appName SOURCE_FILES SOURCE_LIBS) ) endmacro() -add_catch_test(hnsw_test SystemTest/HNSWTest.cpp CANDY) -add_catch_test(cpp_test SystemTest/SimpleTest.cpp CANDY) -add_catch_test(flatIndex_test SystemTest/FlatIndexTest.cpp CANDY) -add_catch_test(flatAMMIPIndex_test SystemTest/FlatAMMIPIndexTest.cpp CANDY) -add_catch_test(flatAMMIPObjIndex_test SystemTest/FlatAMMIPObjIndexTest.cpp CANDY) -add_catch_test(ppIndex_test SystemTest/ParallelPartitionIndexTest.cpp CANDY) -add_catch_test(pqIndex_Test SystemTest/PQIndexTest.cpp CANDY) -add_catch_test(onlinePQ_test SystemTest/OnlinePQIndexTest.cpp CANDY) -add_catch_test(onlineIVFLSH_test SystemTest/OnlineIVFLSHIndexTest.cpp CANDY) -add_catch_test(yinYang_test SystemTest/YinYangGraphIndexTest.cpp CANDY) -add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CANDY) -add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDY) -add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDY) -add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDY) -add_catch_test(threshold_test SystemTest/ThresholdTest.cpp CANDY) + add_catch_test(hnsw_test SystemTest/HNSWTest.cpp CANDYBENCH) add_catch_test(cpp_test SystemTest/SimpleTest.cpp CANDYBENCH) From 9d8b1fc6c3bbd164681d46e934e8adee0cad7609 Mon Sep 17 00:00:00 2001 From: Regis Wu <73592273+Rubatozh@users.noreply.github.com> Date: Mon, 16 Dec 2024 20:10:17 +0800 Subject: [PATCH 7/7] Update tests/CMakeLists.txt --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2f198acb6..2dfaf183b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -52,7 +52,7 @@ add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CAN add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDYBENCH) add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDYBENCH) add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDYBENCH) -add_catch_test(threshold_test SystemTest/ThresholdTest.cpp CANDY) +#add_catch_test(threshold_test SystemTest/ThresholdTest.cpp CANDY) add_catch_test(lshAPGIndex_test SystemTest/LSHAPGIndexTest.cpp CANDYBENCH) if (ENABLE_SPTAG) add_catch_test(sptagIndex_test SystemTest/SPTAGIndexTest.cpp CANDYBENCH)