aa

Signed-off-by: sunby <[email protected]>

Makefile

@@ -20,7 +20,7 @@ INSTALL_PATH := $(PWD)/bin
 LIBRARY_PATH := $(PWD)/lib
 PGO_PATH := $(PWD)/configs/pgo
 OS := $(shell uname -s)
-mode = Release
+mode = Debug
 
 use_disk_index = OFF
 ifdef disk_index

internal/core/src/common/Common.cpp

@@ -29,6 +29,8 @@ int64_t LOW_PRIORITY_THREAD_CORE_COEFFICIENT =
 int CPU_NUM = DEFAULT_CPU_NUM;
 int64_t EXEC_EVAL_EXPR_BATCH_SIZE = DEFAULT_EXEC_EVAL_EXPR_BATCH_SIZE;
 
+int64_t COPY_STR_D = 0;
+
 void
 SetIndexSliceSize(const int64_t size) {
     FILE_SLICE_SIZE = size << 20;

internal/core/src/common/Common.h

@@ -30,6 +30,8 @@ extern int64_t LOW_PRIORITY_THREAD_CORE_COEFFICIENT;
 extern int CPU_NUM;
 extern int64_t EXEC_EVAL_EXPR_BATCH_SIZE;
 
+extern int64_t COPY_STR_D;
+
 void
 SetIndexSliceSize(const int64_t size);
 

internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp

@@ -1440,7 +1440,16 @@ ChunkedSegmentSealedImpl::bulk_subscript_ptr_impl(
     auto field = reinterpret_cast<const ChunkedVariableColumn<S>*>(column);
     for (int64_t i = 0; i < count; ++i) {
         auto offset = seg_offsets[i];
-        dst->at(i) = std::move(T(field->RawAt(offset)));
+        std::string_view t = field->RawAt(offset);
+        std::chrono::high_resolution_clock::time_point start =
+            std::chrono::high_resolution_clock::now();
+        dst->at(i) = std::move(T(t));
+
+        std::chrono::high_resolution_clock::time_point end =
+            std::chrono::high_resolution_clock::now();
+        COPY_STR_D +=
+            std::chrono::duration_cast<std::chrono::microseconds>(end - start)
+                .count();
     }
 }
 

internal/core/src/segcore/ChunkedSegmentSealedImpl.h

@@ -234,7 +234,7 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
         return insert_record_.timestamps_;
     }
 
- private:
+ public:
     template <typename S, typename T = S>
     static void
     bulk_subscript_impl(const void* src_raw,

internal/core/src/segcore/SegmentSealedImpl.cpp

@@ -16,6 +16,7 @@
 #include <sys/stat.h>
 
 #include <algorithm>
+#include <chrono>
 #include <cstdint>
 #include <filesystem>
 #include <memory>
@@ -29,6 +30,7 @@
 #include "Utils.h"
 #include "Types.h"
 #include "common/Array.h"
+#include "common/Common.h"
 #include "common/Consts.h"
 #include "common/EasyAssert.h"
 #include "common/FieldData.h"
@@ -1345,7 +1347,15 @@ SegmentSealedImpl::bulk_subscript_ptr_impl(
     auto field = reinterpret_cast<const SingleChunkVariableColumn<S>*>(column);
     for (int64_t i = 0; i < count; ++i) {
         auto offset = seg_offsets[i];
-        dst->at(i) = std::move(T(field->RawAt(offset)));
+        std::string_view t = field->RawAt(offset);
+        std::chrono::high_resolution_clock::time_point start =
+            std::chrono::high_resolution_clock::now();
+        dst->at(i) = std::move(T(t));
+        std::chrono::high_resolution_clock::time_point end =
+            std::chrono::high_resolution_clock::now();
+        COPY_STR_D +=
+            std::chrono::duration_cast<std::chrono::microseconds>(end - start)
+                .count();
     }
 }
 

internal/core/src/segcore/SegmentSealedImpl.h

@@ -240,7 +240,7 @@ class SegmentSealedImpl : public SegmentSealed {
         return insert_record_.timestamps_;
     }
 
- private:
+ public:
     template <typename S, typename T = S>
     static void
     bulk_subscript_impl(const void* src_raw,

internal/core/unittest/CMakeLists.txt

@@ -24,69 +24,69 @@ add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/te
 set(MILVUS_TEST_FILES
         init_gtest.cpp
 
-        test_always_true_expr.cpp
-        test_array_bitmap_index.cpp
-        test_array_inverted_index.cpp
-        test_bf.cpp
-        test_bf_sparse.cpp
-        test_binary.cpp
-        test_binlog_index.cpp
-        test_bitmap_index.cpp
-        test_bool_index.cpp
-        test_c_api.cpp
-        test_chunk_cache.cpp
-        test_chunk.cpp
-        test_chunk_vector.cpp
-        test_common.cpp
-        test_concurrent_vector.cpp
-        test_c_stream_reduce.cpp
-        test_c_tokenizer.cpp
-        test_loading.cpp
-        test_data_codec.cpp
-        test_disk_file_manager_test.cpp
-        test_exec.cpp
-        test_expr.cpp
-        test_expr_materialized_view.cpp
-        test_float16.cpp
-        test_function.cpp
-        test_futures.cpp
-        test_group_by.cpp
-        test_growing.cpp
-        test_growing_index.cpp
-        test_hybrid_index.cpp
-        test_index_c_api.cpp
-        test_indexing.cpp
-        test_index_wrapper.cpp
-        test_init.cpp
-        test_integer_overflow.cpp
-        test_inverted_index.cpp
-        test_local_chunk_manager.cpp
-        test_mmap_chunk_manager.cpp
-        test_monitor.cpp
-        test_offset_ordered_array.cpp
-        test_offset_ordered_map.cpp
-        test_plan_proto.cpp
-        test_query.cpp
-        test_range_search_sort.cpp
-        test_reduce_c.cpp
-        test_reduce.cpp
-        test_regex_query.cpp
-        test_regex_query_util.cpp
-        test_relational.cpp
-        test_retrieve.cpp
-        test_scalar_index.cpp
-        test_sealed.cpp
-        test_segcore.cpp
-        test_similarity_corelation.cpp
-        test_span.cpp
-        test_storage.cpp
-        test_string_expr.cpp
-        test_text_match.cpp
-        test_timestamp_index.cpp
-        test_tracer.cpp
-        test_utils.cpp
+        # test_always_true_expr.cpp
+        # test_array_bitmap_index.cpp
+        # test_array_inverted_index.cpp
+        # test_bf.cpp
+        # test_bf_sparse.cpp
+        # test_binary.cpp
+        # test_binlog_index.cpp
+        # test_bitmap_index.cpp
+        # test_bool_index.cpp
+        # test_c_api.cpp
+        # test_chunk_cache.cpp
+        # test_chunk.cpp
+        # test_chunk_vector.cpp
+        # test_common.cpp
+        # test_concurrent_vector.cpp
+        # test_c_stream_reduce.cpp
+        # test_c_tokenizer.cpp
+        # test_loading.cpp
+        # test_data_codec.cpp
+        # test_disk_file_manager_test.cpp
+        # test_exec.cpp
+        # test_expr.cpp
+        # test_expr_materialized_view.cpp
+        # test_float16.cpp
+        # test_function.cpp
+        # test_futures.cpp
+        # test_group_by.cpp
+        # test_growing.cpp
+        # test_growing_index.cpp
+        # test_hybrid_index.cpp
+        # test_index_c_api.cpp
+        # test_indexing.cpp
+        # test_index_wrapper.cpp
+        # test_init.cpp
+        # test_integer_overflow.cpp
+        # test_inverted_index.cpp
+        # test_local_chunk_manager.cpp
+        # test_mmap_chunk_manager.cpp
+        # test_monitor.cpp
+        # test_offset_ordered_array.cpp
+        # test_offset_ordered_map.cpp
+        # test_plan_proto.cpp
+        # test_query.cpp
+        # test_range_search_sort.cpp
+        # test_reduce_c.cpp
+        # test_reduce.cpp
+        # test_regex_query.cpp
+        # test_regex_query_util.cpp
+        # test_relational.cpp
+        # test_retrieve.cpp
+        # test_scalar_index.cpp
+        # test_sealed.cpp
+        # test_segcore.cpp
+        # test_similarity_corelation.cpp
+        # test_span.cpp
+        # test_storage.cpp
+        # test_string_expr.cpp
+        # test_text_match.cpp
+        # test_timestamp_index.cpp
+        # test_tracer.cpp
+        # test_utils.cpp
         test_chunked_segment.cpp
-        test_chunked_column.cpp
+        # test_chunked_column.cpp
         )
 
 if ( INDEX_ENGINE STREQUAL "cardinal" )
@@ -103,23 +103,23 @@ if ( BUILD_DISK_ANN STREQUAL "ON" )
             )
 endif()
 
-if (LINUX OR APPLE)
-    set(MILVUS_TEST_FILES
-            ${MILVUS_TEST_FILES}
-            test_scalar_index_creator.cpp
-            test_string_index.cpp
-            test_array.cpp test_array_expr.cpp)
-endif()
+# if (LINUX OR APPLE)
+#     set(MILVUS_TEST_FILES
+#             ${MILVUS_TEST_FILES}
+#             test_scalar_index_creator.cpp
+#             test_string_index.cpp
+#             test_array.cpp test_array_expr.cpp)
+# endif()
 
-if (DEFINED AZURE_BUILD_DIR)
-    set(MILVUS_TEST_FILES
-                ${MILVUS_TEST_FILES}
-                test_azure_chunk_manager.cpp
-            #need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
-                #test_remote_chunk_manager.cpp
-                )
-    include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include")
-endif()
+# if (DEFINED AZURE_BUILD_DIR)
+#     set(MILVUS_TEST_FILES
+#                 ${MILVUS_TEST_FILES}
+#                 test_azure_chunk_manager.cpp
+#             #need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
+#                 #test_remote_chunk_manager.cpp
+#                 )
+#     include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include")
+# endif()
 
 if (ENABLE_GCP_NATIVE)
     add_definitions(-DENABLE_GCP_NATIVE)
@@ -129,33 +129,33 @@ if (ENABLE_GCP_NATIVE)
                 )
 endif()
 
-if (LINUX)
-    message( STATUS "Building Milvus Unit Test on Linux")
-    option(USE_ASAN "Whether to use AddressSanitizer" OFF)
-    if ( USE_ASAN )
-        message( STATUS "Building Milvus using AddressSanitizer")
-        add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
-        add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
-    endif()
-
-    # check if memory leak exists in index builder
-    set(INDEX_BUILDER_TEST_FILES
-            test_index_wrapper.cpp
-            test_scalar_index_creator.cpp
-            test_index_c_api.cpp
-            )
-
-    add_executable(index_builder_test
-            ${INDEX_BUILDER_TEST_FILES}
-            )
-
-    target_link_libraries(index_builder_test
-            gtest
-            milvus_core
-            knowhere
-            )
-    install(TARGETS index_builder_test DESTINATION unittest)
-endif()
+# if (LINUX)
+#     message( STATUS "Building Milvus Unit Test on Linux")
+#     option(USE_ASAN "Whether to use AddressSanitizer" OFF)
+#     if ( USE_ASAN )
+#         message( STATUS "Building Milvus using AddressSanitizer")
+#         add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
+#         add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
+#     endif()
+
+#     # check if memory leak exists in index builder
+#     set(INDEX_BUILDER_TEST_FILES
+#             test_index_wrapper.cpp
+#             test_scalar_index_creator.cpp
+#             test_index_c_api.cpp
+#             )
+
+#     add_executable(index_builder_test
+#             ${INDEX_BUILDER_TEST_FILES}
+#             )
+
+#     target_link_libraries(index_builder_test
+#             gtest
+#             milvus_core
+#             knowhere
+#             )
+#     install(TARGETS index_builder_test DESTINATION unittest)
+# endif()
 
 add_executable(all_tests
         ${MILVUS_TEST_FILES}
@@ -169,9 +169,9 @@ target_link_libraries(all_tests
 
 install(TARGETS all_tests DESTINATION unittest)
 
-if (LINUX)
-    add_subdirectory(bench)
-endif ()
+# if (LINUX)
+#     add_subdirectory(bench)
+# endif ()
 
 # if (USE_DYNAMIC_SIMD)
 # add_executable(dynamic_simd_test