Skip to content

Commit

Permalink
Merge pull request #55 from intellistream/benchPython
Browse files Browse the repository at this point in the history 
1. resolve name conflicts
  • Loading branch information
@TONYSKYZENG
TONYSKYZENG authored Nov 4, 2024
2 parents 0988e3d + b671b3e commit 82864fa
Show file tree
Hide file tree
Showing 6 changed files with 58 additions and 47 deletions.
32 changes: 16 additions & 16 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

cmake_minimum_required(VERSION 3.14)
project(CANDY CXX)
project(CANDYBENCH CXX)
include (cmake/FindCuda.cmake)
include (cmake/FindTorch.cmake)
find_package(Torch REQUIRED)
Expand Down Expand Up @@ -78,7 +78,7 @@ option(ENABLE_SPTAG
OFF
)
option(ENABLE_PYBIND
"Enable original pybind and build CANDY python"
"Enable original pybind and build CANDYBENCH python"
OFF
)

Expand Down Expand Up @@ -221,19 +221,19 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/)
# Add Library
get_sources(CANDY_SOURCE_FILES)
get_headers(CANDY_HEADER_FILES)
add_library(CANDY SHARED ${CANDY_SOURCE_FILES} ${CANDY_HEADER_FILES} ${CMAKE_CURRENT_BINARY_DIR})
add_library(CANDYBENCH SHARED ${CANDY_SOURCE_FILES} ${CANDY_HEADER_FILES} ${CMAKE_CURRENT_BINARY_DIR})

set_property(TARGET CANDY PROPERTY CXX_STANDARD 20)
target_include_directories(CANDY PUBLIC "include")
target_include_directories(CANDY PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
target_include_directories(CANDY PUBLIC "thirdparty/papi_build/include")
target_include_directories(CANDY PUBLIC "thirdparty/")
target_include_directories(CANDY PUBLIC "thirdparty/SPTAG")
target_include_directories(CANDY PUBLIC "thirdparty/SPTAG/AnnService")
target_link_options(CANDY PUBLIC "-Wl,-rpath,./")
target_link_options(CANDY PUBLIC "-Wl,-rpath,./")
target_link_libraries(CANDY PUBLIC ${LIBRARIES})
install(TARGETS CANDY LIBRARY DESTINATION .)
set_property(TARGET CANDYBENCH PROPERTY CXX_STANDARD 20)
target_include_directories(CANDYBENCH PUBLIC "include")
target_include_directories(CANDYBENCH PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
target_include_directories(CANDYBENCH PUBLIC "thirdparty/papi_build/include")
target_include_directories(CANDYBENCH PUBLIC "thirdparty/")
target_include_directories(CANDYBENCH PUBLIC "thirdparty/SPTAG")
target_include_directories(CANDYBENCH PUBLIC "thirdparty/SPTAG/AnnService")
target_link_options(CANDYBENCH PUBLIC "-Wl,-rpath,./")
target_link_options(CANDYBENCH PUBLIC "-Wl,-rpath,./")
target_link_libraries(CANDYBENCH PUBLIC ${LIBRARIES})
install(TARGETS CANDYBENCH LIBRARY DESTINATION .)
#add_subdirectory(test)
#Add benchmarks with command
add_subdirectory(benchmark)
Expand All @@ -252,7 +252,7 @@ else ()
add_subdirectory(thirdparty/pybind11)
pybind11_add_module(PyCANDY ${PROJECT_SOURCE_DIR}/src/PyCANDY.cpp)
find_library(TORCH_PYTHON_LIBRARY torch_python PATH "${TORCH_INSTALL_PREFIX}/lib")
target_link_libraries(PyCANDY PUBLIC ${LIBRARIES} CANDY ${TORCH_PYTHON_LIBRARY})
target_link_libraries(PyCANDY PUBLIC ${LIBRARIES} CANDYBENCH ${TORCH_PYTHON_LIBRARY})
install(TARGETS PyCANDY LIBRARY DESTINATION .)
message(STATUS "Additional include torch_python: ${TORCH_PYTHON_LIBRARY}")
set(CANDY_PYBIND 1)
Expand All @@ -262,7 +262,7 @@ configure_file(
"${PROJECT_SOURCE_DIR}/include/pybind_config.h.in"
"${PROJECT_BINARY_DIR}/include/pybind_config.h"
)
install(DIRECTORY "include" DESTINATION "/CANDY" COMPONENT CANDY)
install(DIRECTORY "include" DESTINATION "/CANDY" COMPONENT CANDYBENCH)
# copy scripts
file(GLOB allCopyFiles "${PROJECT_SOURCE_DIR}/scripts/*")
file(COPY ${allCopyFiles} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/benchmark/scripts)
Expand Down
18 changes: 9 additions & 9 deletions benchmark/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,33 +19,33 @@ if (BUILD_BENCHMARKS)
include_directories("${source_dir}/include/")
include_directories("include/")
add_executable(onlineInsert "src/OnlineInsert.cpp")
target_link_libraries(onlineInsert CANDY)
target_link_libraries(onlineInsert CANDYBENCH)
add_executable(offlineBuild "src/OfflineBuild.cpp")
target_link_libraries(offlineBuild CANDY)
target_link_libraries(offlineBuild CANDYBENCH)
add_executable(multiRW "src/MultiRW.cpp")
target_link_libraries(multiRW CANDY)
target_link_libraries(multiRW CANDYBENCH)
add_executable(onlineInsert2 "src/OnlineInsertLSHAPG.cpp")
target_link_libraries(onlineInsert2 CANDY)
target_link_libraries(onlineInsert2 CANDYBENCH)
add_executable(onlineCUD "src/OnlineCUD.cpp")
target_link_libraries(onlineCUD CANDY)
target_link_libraries(onlineCUD CANDYBENCH)
if (ENABLE_RAY)
add_library(onlineInsertL SHARED "src/OnlineInsert.cpp")
target_link_libraries(onlineInsertL CANDY)
target_link_libraries(onlineInsertL CANDYBENCH)
target_link_options(onlineInsert PRIVATE "-Wl,-rpath,./")
target_link_options(onlineInsertL PRIVATE "-Wl,-rpath,./")
#
add_library(offlineBuildL SHARED "src/OfflineBuild.cpp")
target_link_libraries(offlineBuildL CANDY)
target_link_libraries(offlineBuildL CANDYBENCH)
target_link_options(offlineBuild PRIVATE "-Wl,-rpath,./")
target_link_options(offlineBuildL PRIVATE "-Wl,-rpath,./")

add_library(multiRWL SHARED "src/MultiRW.cpp")
target_link_libraries(multiRWL CANDY)
target_link_libraries(multiRWL CANDYBENCH)
target_link_options(multiRWL PRIVATE "-Wl,-rpath,./")
target_link_options(multiRW PRIVATE "-Wl,-rpath,./")

add_library(onlineCUDL SHARED "src/OnlineCUD.cpp")
target_link_libraries(onlineCUDL CANDY)
target_link_libraries(onlineCUDL CANDYBENCH)
target_link_options(onlineCUDL PRIVATE "-Wl,-rpath,./")
target_link_options(onlineCUDL PRIVATE "-Wl,-rpath,./")

Expand Down
12 changes: 10 additions & 2 deletions benchmark/scripts/PyCANDY/benchmark_tool/mainWindow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ def run(self):
eventRate = int(self.config_map.get("eventRateTps", 1000))
deleteRows = int(self.config_map.get("deleteRows", -1))
querySize = int(self.config_map.get("querySize", 100))
vecVolume = int(self.config_map.get("memBufferSize",10000 ))
memBufferSize = int(self.config_map.get("memBufferSize",vecVolume ))
self.querySize = querySize
self.deleteRows = deleteRows
annk = int(self.config_map.get("ANNK", 10))
Expand Down Expand Up @@ -120,6 +122,7 @@ def __init__(self):
"result": "Final Result",
"toggle_output": "Show/Hide Intermediate Output",
"index_tag":"Name of Algo",
"cuda_idx":"Cuda device id",
"event_rate":"Event Rate"
},
"中文": {
Expand All @@ -141,7 +144,8 @@ def __init__(self):
"result": "最终结果",
"toggle_output": "显示/隐藏中间输出",
"index_tag":"算法名称",
"Event Rate":"事件速率"
"Event Rate":"事件速率",
"cuda_idx":"Cuda设备号",
}
}
self.current_language = "English"
Expand Down Expand Up @@ -189,7 +193,7 @@ def initUI(self):
self.event_rate_input = QLineEdit("-1")
self.batch_size_input = QLineEdit("4000")
self.idx_tag_input = QLineEdit("flat")

self.cuda_input = QLineEdit("-1")
self.param_frame_layout.addRow(self.enable_random_checkbox)
self.param_frame_layout.addRow(self.languages[self.current_language]["index_tag"], self.idx_tag_input)
self.param_frame_layout.addRow(self.languages[self.current_language]["vec_volume"], self.vec_volume_input)
Expand All @@ -201,6 +205,7 @@ def initUI(self):
self.param_frame_layout.addRow(self.languages[self.current_language]["query_size"], self.query_size_input)
self.param_frame_layout.addRow(self.languages[self.current_language]["annk"], self.annk_input)
self.param_frame_layout.addRow(self.languages[self.current_language]["batch_size"], self.batch_size_input)
self.param_frame_layout.addRow(self.languages[self.current_language]["cuda_idx"], self.cuda_input)

# Toggle button for parameter section
self.toggle_button = QPushButton("Show/Hide Parameters")
Expand Down Expand Up @@ -361,6 +366,7 @@ def get_parameters_dict(self):
"ANNK": int(self.annk_input.text()),
"batchSize": int(self.batch_size_input.text()),
"indexTag": (self.idx_tag_input.text()),
"cudaDevice":int(self.cuda_input.text())
}
if(self.enable_random_checkbox.isChecked()):
params['dataLoaderTag']=str('random')
Expand Down Expand Up @@ -402,6 +408,8 @@ def load_parameters_from_dict(self, params):
self.annk_input.setText(str(params["ANNK"]))
if "batchSize" in params:
self.batch_size_input.setText(str(params["batchSize"]))
if "cudaDevice" in params:
self.cuda_input.setText(str(params["cudaDevice"]))


def displayConfigInTable(self, config_dict):
Expand Down
3 changes: 2 additions & 1 deletion src/CANDY/FlatGPUIndex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ bool CANDY::FlatGPUIndex::setConfig(INTELLI::ConfigMapPtr cfg) {
distanceFunc = distanceL2;
}
vecDim = cfg->tryI64("vecDim", 768, true);
memBufferSize = cfg->tryI64("memBufferSize", 1000, true);
int64_t vecVolume = cfg->tryI64("vecVolume", 1000, true);
memBufferSize = cfg->tryI64("memBufferSize", vecVolume+1000, true);
sketchSize = cfg->tryI64("sketchSize", 10, true);
DCOBatchSize = cfg->tryI64("DCOBatchSize", memBufferSize, true);
if (torch::cuda::is_available()) {
Expand Down
2 changes: 2 additions & 0 deletions src/PyCANDY.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,11 +117,13 @@ double recallOfTensorList(std::vector<torch::Tensor> groundTruth, std::vector<to
double recall = static_cast<double>(truePositives) / (truePositives + falseNegatives);
return recall;
}
#define COMPILED_TIME (__DATE__ " " __TIME__)
PYBIND11_MODULE(PyCANDY, m) {
/**
* @brief export the configmap class
*/
m.attr("__version__") = "0.1.2"; // Set the version of the module
m.attr("__compiled_time__") = COMPILED_TIME; // Set the compile time of the module
py::class_<INTELLI::ConfigMap, std::shared_ptr<INTELLI::ConfigMap>>(m, "ConfigMap")
.def(py::init<>())
.def("edit", py::overload_cast<const std::string &, int64_t>(&INTELLI::ConfigMap::edit))
Expand Down
38 changes: 19 additions & 19 deletions test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,30 +35,30 @@ macro(add_catch_test_ray appName SOURCE_FILES SOURCE_LIBS)
COMMAND $<TARGET_FILE:${appName}> --success
)
endmacro()
add_catch_test(hnsw_test SystemTest/HNSWTest.cpp CANDY)
add_catch_test(cpp_test SystemTest/SimpleTest.cpp CANDY)
add_catch_test(flatIndex_test SystemTest/FlatIndexTest.cpp CANDY)
add_catch_test(flatAMMIPIndex_test SystemTest/FlatAMMIPIndexTest.cpp CANDY)
add_catch_test(flatAMMIPObjIndex_test SystemTest/FlatAMMIPObjIndexTest.cpp CANDY)
add_catch_test(ppIndex_test SystemTest/ParallelPartitionIndexTest.cpp CANDY)
add_catch_test(pqIndex_Test SystemTest/PQIndexTest.cpp CANDY)
add_catch_test(onlinePQ_test SystemTest/OnlinePQIndexTest.cpp CANDY)
add_catch_test(onlineIVFLSH_test SystemTest/OnlineIVFLSHIndexTest.cpp CANDY)
add_catch_test(yinYang_test SystemTest/YinYangGraphIndexTest.cpp CANDY)
add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CANDY)
add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDY)
add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDY)
add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDY)
add_catch_test(lshAPGIndex_test SystemTest/LSHAPGIndexTest.cpp CANDY)
add_catch_test(hnsw_test SystemTest/HNSWTest.cpp CANDYBENCH)
add_catch_test(cpp_test SystemTest/SimpleTest.cpp CANDYBENCH)
add_catch_test(flatIndex_test SystemTest/FlatIndexTest.cpp CANDYBENCH)
add_catch_test(flatAMMIPIndex_test SystemTest/FlatAMMIPIndexTest.cpp CANDYBENCH)
add_catch_test(flatAMMIPObjIndex_test SystemTest/FlatAMMIPObjIndexTest.cpp CANDYBENCH)
add_catch_test(ppIndex_test SystemTest/ParallelPartitionIndexTest.cpp CANDYBENCH)
add_catch_test(pqIndex_Test SystemTest/PQIndexTest.cpp CANDYBENCH)
add_catch_test(onlinePQ_test SystemTest/OnlinePQIndexTest.cpp CANDYBENCH)
add_catch_test(onlineIVFLSH_test SystemTest/OnlineIVFLSHIndexTest.cpp CANDYBENCH)
add_catch_test(yinYang_test SystemTest/YinYangGraphIndexTest.cpp CANDYBENCH)
add_catch_test(yinYangSimple_test SystemTest/YinYangGraphSimpleIndexTest.cpp CANDYBENCH)
add_catch_test(nnDescent_test SystemTest/NNDescentIndexTest.cpp CANDYBENCH)
add_catch_test(kdTree_test SystemTest/KdTreeTest.cpp CANDYBENCH)
add_catch_test(dpgIndex_test SystemTest/DPGIndexTest.cpp CANDYBENCH)
add_catch_test(lshAPGIndex_test SystemTest/LSHAPGIndexTest.cpp CANDYBENCH)
if (ENABLE_SPTAG)
add_catch_test(sptagIndex_test SystemTest/SPTAGIndexTest.cpp CANDY)
add_catch_test(sptagIndex_test SystemTest/SPTAGIndexTest.cpp CANDYBENCH)
endif ()

if (ENABLE_OPENCL)
add_catch_test(cl_test SystemTest/CLTest.cpp CANDY)
add_catch_test(cl_test SystemTest/CLTest.cpp CANDYBENCH)
endif ()
if (ENABLE_RAY)
add_catch_test(dp_test SystemTest/DistributedPartitionIndexTest.cpp CANDY)
add_catch_test_ray(dp_testL SystemTest/DistributedPartitionIndexTest.cpp CANDY)
add_catch_test(dp_test SystemTest/DistributedPartitionIndexTest.cpp CANDYBENCH)
add_catch_test_ray(dp_testL SystemTest/DistributedPartitionIndexTest.cpp CANDYBENCH)
endif ()

0 comments on commit 82864fa

Please sign in to comment.