From b671b3e9ba7eec942814eebd4c997ed7be379efe Mon Sep 17 00:00:00 2001 From: tonyskyzeng <292224750@qq.com> Date: Mon, 4 Nov 2024 13:48:51 +0800 Subject: [PATCH] 1. resolve name conflicts --- CMakeLists.txt | 32 ++++++++-------- benchmark/CMakeLists.txt | 18 ++++----- .../PyCANDY/benchmark_tool/mainWindow.py | 12 +++++- src/CANDY/FlatGPUIndex.cpp | 3 +- src/PyCANDY.cpp | 2 + test/CMakeLists.txt | 38 +++++++++---------- 6 files changed, 58 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 96430bc02..8aa08070a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.14) -project(CANDY CXX) +project(CANDYBENCH CXX) include (cmake/FindCuda.cmake) include (cmake/FindTorch.cmake) find_package(Torch REQUIRED) @@ -78,7 +78,7 @@ option(ENABLE_SPTAG OFF ) option(ENABLE_PYBIND - "Enable original pybind and build CANDY python" + "Enable original pybind and build CANDYBENCH python" OFF ) @@ -221,19 +221,19 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) # Add Library get_sources(CANDY_SOURCE_FILES) get_headers(CANDY_HEADER_FILES) -add_library(CANDY SHARED ${CANDY_SOURCE_FILES} ${CANDY_HEADER_FILES} ${CMAKE_CURRENT_BINARY_DIR}) +add_library(CANDYBENCH SHARED ${CANDY_SOURCE_FILES} ${CANDY_HEADER_FILES} ${CMAKE_CURRENT_BINARY_DIR}) -set_property(TARGET CANDY PROPERTY CXX_STANDARD 20) -target_include_directories(CANDY PUBLIC "include") -target_include_directories(CANDY PUBLIC "${CMAKE_CURRENT_BINARY_DIR}") -target_include_directories(CANDY PUBLIC "thirdparty/papi_build/include") -target_include_directories(CANDY PUBLIC "thirdparty/") -target_include_directories(CANDY PUBLIC "thirdparty/SPTAG") -target_include_directories(CANDY PUBLIC "thirdparty/SPTAG/AnnService") -target_link_options(CANDY PUBLIC "-Wl,-rpath,./") -target_link_options(CANDY PUBLIC "-Wl,-rpath,./") -target_link_libraries(CANDY PUBLIC ${LIBRARIES}) -install(TARGETS CANDY LIBRARY DESTINATION .) +set_property(TARGET CANDYBENCH PROPERTY CXX_STANDARD 20) +target_include_directories(CANDYBENCH PUBLIC "include") +target_include_directories(CANDYBENCH PUBLIC "${CMAKE_CURRENT_BINARY_DIR}") +target_include_directories(CANDYBENCH PUBLIC "thirdparty/papi_build/include") +target_include_directories(CANDYBENCH PUBLIC "thirdparty/") +target_include_directories(CANDYBENCH PUBLIC "thirdparty/SPTAG") +target_include_directories(CANDYBENCH PUBLIC "thirdparty/SPTAG/AnnService") +target_link_options(CANDYBENCH PUBLIC "-Wl,-rpath,./") +target_link_options(CANDYBENCH PUBLIC "-Wl,-rpath,./") +target_link_libraries(CANDYBENCH PUBLIC ${LIBRARIES}) +install(TARGETS CANDYBENCH LIBRARY DESTINATION .) #add_subdirectory(test) #Add benchmarks with command add_subdirectory(benchmark) @@ -252,7 +252,7 @@ else () add_subdirectory(thirdparty/pybind11) pybind11_add_module(PyCANDY ${PROJECT_SOURCE_DIR}/src/PyCANDY.cpp) find_library(TORCH_PYTHON_LIBRARY torch_python PATH "${TORCH_INSTALL_PREFIX}/lib") - target_link_libraries(PyCANDY PUBLIC ${LIBRARIES} CANDY ${TORCH_PYTHON_LIBRARY}) + target_link_libraries(PyCANDY PUBLIC ${LIBRARIES} CANDYBENCH ${TORCH_PYTHON_LIBRARY}) install(TARGETS PyCANDY LIBRARY DESTINATION .) message(STATUS "Additional include torch_python: ${TORCH_PYTHON_LIBRARY}") set(CANDY_PYBIND 1) @@ -262,7 +262,7 @@ configure_file( "${PROJECT_SOURCE_DIR}/include/pybind_config.h.in" "${PROJECT_BINARY_DIR}/include/pybind_config.h" ) -install(DIRECTORY "include" DESTINATION "/CANDY" COMPONENT CANDY) +install(DIRECTORY "include" DESTINATION "/CANDY" COMPONENT CANDYBENCH) # copy scripts file(GLOB allCopyFiles "${PROJECT_SOURCE_DIR}/scripts/*") file(COPY ${allCopyFiles} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/benchmark/scripts) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index e93b35d50..1e070b10f 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -19,33 +19,33 @@ if (BUILD_BENCHMARKS) include_directories("${source_dir}/include/") include_directories("include/") add_executable(onlineInsert "src/OnlineInsert.cpp") - target_link_libraries(onlineInsert CANDY) + target_link_libraries(onlineInsert CANDYBENCH) add_executable(offlineBuild "src/OfflineBuild.cpp") - target_link_libraries(offlineBuild CANDY) + target_link_libraries(offlineBuild CANDYBENCH) add_executable(multiRW "src/MultiRW.cpp") - target_link_libraries(multiRW CANDY) + target_link_libraries(multiRW CANDYBENCH) add_executable(onlineInsert2 "src/OnlineInsertLSHAPG.cpp") - target_link_libraries(onlineInsert2 CANDY) + target_link_libraries(onlineInsert2 CANDYBENCH) add_executable(onlineCUD "src/OnlineCUD.cpp") - target_link_libraries(onlineCUD CANDY) + target_link_libraries(onlineCUD CANDYBENCH) if (ENABLE_RAY) add_library(onlineInsertL SHARED "src/OnlineInsert.cpp") - target_link_libraries(onlineInsertL CANDY) + target_link_libraries(onlineInsertL CANDYBENCH) target_link_options(onlineInsert PRIVATE "-Wl,-rpath,./") target_link_options(onlineInsertL PRIVATE "-Wl,-rpath,./") # add_library(offlineBuildL SHARED "src/OfflineBuild.cpp") - target_link_libraries(offlineBuildL CANDY) + target_link_libraries(offlineBuildL CANDYBENCH) target_link_options(offlineBuild PRIVATE "-Wl,-rpath,./") target_link_options(offlineBuildL PRIVATE "-Wl,-rpath,./") add_library(multiRWL SHARED "src/MultiRW.cpp") - target_link_libraries(multiRWL CANDY) + target_link_libraries(multiRWL CANDYBENCH) target_link_options(multiRWL PRIVATE "-Wl,-rpath,./") target_link_options(multiRW PRIVATE "-Wl,-rpath,./") add_library(onlineCUDL SHARED "src/OnlineCUD.cpp") - target_link_libraries(onlineCUDL CANDY) + target_link_libraries(onlineCUDL CANDYBENCH) target_link_options(onlineCUDL PRIVATE "-Wl,-rpath,./") target_link_options(onlineCUDL PRIVATE "-Wl,-rpath,./") diff --git a/benchmark/scripts/PyCANDY/benchmark_tool/mainWindow.py b/benchmark/scripts/PyCANDY/benchmark_tool/mainWindow.py index 76fca58d9..86420722e 100644 --- a/benchmark/scripts/PyCANDY/benchmark_tool/mainWindow.py +++ b/benchmark/scripts/PyCANDY/benchmark_tool/mainWindow.py @@ -42,6 +42,8 @@ def run(self): eventRate = int(self.config_map.get("eventRateTps", 1000)) deleteRows = int(self.config_map.get("deleteRows", -1)) querySize = int(self.config_map.get("querySize", 100)) + vecVolume = int(self.config_map.get("memBufferSize",10000 )) + memBufferSize = int(self.config_map.get("memBufferSize",vecVolume )) self.querySize = querySize self.deleteRows = deleteRows annk = int(self.config_map.get("ANNK", 10)) @@ -120,6 +122,7 @@ def __init__(self): "result": "Final Result", "toggle_output": "Show/Hide Intermediate Output", "index_tag":"Name of Algo", + "cuda_idx":"Cuda device id", "event_rate":"Event Rate" }, "中文": { @@ -141,7 +144,8 @@ def __init__(self): "result": "最终结果", "toggle_output": "显示/隐藏中间输出", "index_tag":"算法名称", - "Event Rate":"事件速率" + "Event Rate":"事件速率", + "cuda_idx":"Cuda设备号", } } self.current_language = "English" @@ -189,7 +193,7 @@ def initUI(self): self.event_rate_input = QLineEdit("-1") self.batch_size_input = QLineEdit("4000") self.idx_tag_input = QLineEdit("flat") - + self.cuda_input = QLineEdit("-1") self.param_frame_layout.addRow(self.enable_random_checkbox) self.param_frame_layout.addRow(self.languages[self.current_language]["index_tag"], self.idx_tag_input) self.param_frame_layout.addRow(self.languages[self.current_language]["vec_volume"], self.vec_volume_input) @@ -201,6 +205,7 @@ def initUI(self): self.param_frame_layout.addRow(self.languages[self.current_language]["query_size"], self.query_size_input) self.param_frame_layout.addRow(self.languages[self.current_language]["annk"], self.annk_input) self.param_frame_layout.addRow(self.languages[self.current_language]["batch_size"], self.batch_size_input) + self.param_frame_layout.addRow(self.languages[self.current_language]["cuda_idx"], self.cuda_input) # Toggle button for parameter section self.toggle_button = QPushButton("Show/Hide Parameters") @@ -361,6 +366,7 @@ def get_parameters_dict(self): "ANNK": int(self.annk_input.text()), "batchSize": int(self.batch_size_input.text()), "indexTag": (self.idx_tag_input.text()), + "cudaDevice":int(self.cuda_input.text()) } if(self.enable_random_checkbox.isChecked()): params['dataLoaderTag']=str('random') @@ -402,6 +408,8 @@ def load_parameters_from_dict(self, params): self.annk_input.setText(str(params["ANNK"])) if "batchSize" in params: self.batch_size_input.setText(str(params["batchSize"])) + if "cudaDevice" in params: + self.cuda_input.setText(str(params["cudaDevice"])) def displayConfigInTable(self, config_dict): diff --git a/src/CANDY/FlatGPUIndex.cpp b/src/CANDY/FlatGPUIndex.cpp index 8082e577c..23daa51ce 100644 --- a/src/CANDY/FlatGPUIndex.cpp +++ b/src/CANDY/FlatGPUIndex.cpp @@ -26,7 +26,8 @@ bool CANDY::FlatGPUIndex::setConfig(INTELLI::ConfigMapPtr cfg) { distanceFunc = distanceL2; } vecDim = cfg->tryI64("vecDim", 768, true); - memBufferSize = cfg->tryI64("memBufferSize", 1000, true); + int64_t vecVolume = cfg->tryI64("vecVolume", 1000, true); + memBufferSize = cfg->tryI64("memBufferSize", vecVolume+1000, true); sketchSize = cfg->tryI64("sketchSize", 10, true); DCOBatchSize = cfg->tryI64("DCOBatchSize", memBufferSize, true); if (torch::cuda::is_available()) { diff --git a/src/PyCANDY.cpp b/src/PyCANDY.cpp index a7d73a17c..411c17f72 100644 --- a/src/PyCANDY.cpp +++ b/src/PyCANDY.cpp @@ -117,11 +117,13 @@ double recallOfTensorList(std::vector groundTruth, std::vector(truePositives) / (truePositives + falseNegatives); return recall; } +#define COMPILED_TIME (__DATE__ " " __TIME__) PYBIND11_MODULE(PyCANDY, m) { /** * @brief export the configmap class */ m.attr("__version__") = "0.1.2"; // Set the version of the module + m.attr("__compiled_time__") = COMPILED_TIME; // Set the compile time of the module py::class_>(m, "ConfigMap") .def(py::init<>()) .def("edit", py::overload_cast(&INTELLI::ConfigMap::edit)) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 934a0e1c3..c1a393aff 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,30 +35,30 @@ macro(add_catch_test_ray appName SOURCE_FILES SOURCE_LIBS) COMMAND $ --success ) endmacro() -add_catch_test(hnsw_test SystemTest/HNSWTest.cpp CANDY) -add_catch_test(cpp_test SystemTest/SimpleTest.cpp CANDY) -add_catch_test(flatIndex_test SystemTest/FlatIndexTest.cpp CANDY) -add_catch_test(flatAMMIPIndex_test SystemTest/FlatAMMIPIndexTest.cpp CANDY) -add_catch_test(flatAMMIPObjIndex_test SystemTest/FlatAMMIPObjIndexTest.cpp CANDY) -add_catch_test(ppIndex_test SystemTest/ParallelPartitionIndexTest.cpp CANDY) -add_catch_test(pqIndex_Test SystemTest/PQIndexTest.cpp CANDY) -add_catch_test(onlinePQ_test SystemTest/OnlinePQIndexTest.cpp CANDY) -add_catch_test(onlineIVFLSH_test SystemTest/OnlineIVFLSHIndexTest.cpp CANDY) -add_catch_test(yinYang_test SystemTest/YinYangGraphIndexTest.cpp CANDY) -add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CANDY) -add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDY) -add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDY) -add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDY) -add_catch_test(lshAPGIndex_test SystemTest/LSHAPGIndexTest.cpp CANDY) +add_catch_test(hnsw_test SystemTest/HNSWTest.cpp CANDYBENCH) +add_catch_test(cpp_test SystemTest/SimpleTest.cpp CANDYBENCH) +add_catch_test(flatIndex_test SystemTest/FlatIndexTest.cpp CANDYBENCH) +add_catch_test(flatAMMIPIndex_test SystemTest/FlatAMMIPIndexTest.cpp CANDYBENCH) +add_catch_test(flatAMMIPObjIndex_test SystemTest/FlatAMMIPObjIndexTest.cpp CANDYBENCH) +add_catch_test(ppIndex_test SystemTest/ParallelPartitionIndexTest.cpp CANDYBENCH) +add_catch_test(pqIndex_Test SystemTest/PQIndexTest.cpp CANDYBENCH) +add_catch_test(onlinePQ_test SystemTest/OnlinePQIndexTest.cpp CANDYBENCH) +add_catch_test(onlineIVFLSH_test SystemTest/OnlineIVFLSHIndexTest.cpp CANDYBENCH) +add_catch_test(yinYang_test SystemTest/YinYangGraphIndexTest.cpp CANDYBENCH) +add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CANDYBENCH) +add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDYBENCH) +add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDYBENCH) +add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDYBENCH) +add_catch_test(lshAPGIndex_test SystemTest/LSHAPGIndexTest.cpp CANDYBENCH) if (ENABLE_SPTAG) - add_catch_test(sptagIndex_test SystemTest/SPTAGIndexTest.cpp CANDY) + add_catch_test(sptagIndex_test SystemTest/SPTAGIndexTest.cpp CANDYBENCH) endif () if (ENABLE_OPENCL) - add_catch_test(cl_test SystemTest/CLTest.cpp CANDY) + add_catch_test(cl_test SystemTest/CLTest.cpp CANDYBENCH) endif () if (ENABLE_RAY) - add_catch_test(dp_test SystemTest/DistributedPartitionIndexTest.cpp CANDY) - add_catch_test_ray(dp_testL SystemTest/DistributedPartitionIndexTest.cpp CANDY) + add_catch_test(dp_test SystemTest/DistributedPartitionIndexTest.cpp CANDYBENCH) + add_catch_test_ray(dp_testL SystemTest/DistributedPartitionIndexTest.cpp CANDYBENCH) endif ()