From b3a5672dbd822266751fd81f4cc478b56209406c Mon Sep 17 00:00:00 2001 From: "zhenshan.cao" Date: Sat, 2 Apr 2022 17:34:46 +0800 Subject: [PATCH] Enhance ConcurrentBitset and BitsetView Signed-off-by: zhenshan.cao --- knowhere/CMakeLists.txt | 12 +- knowhere/index/vector_index/Statistics.h | 2 +- knowhere/utils/Bitset.cpp | 327 ++++++++++++++++++ knowhere/utils/Bitset.h | 131 +++++++ knowhere/utils/BitsetView.cpp | 128 +++++++ knowhere/utils/BitsetView.h | 297 ++-------------- knowhere/utils/CMakeLists.txt | 5 +- thirdparty/faiss/Makefile.tmp | 24 -- thirdparty/faiss/faiss/gpu/GpuIndexIVFFlat.cu | 2 +- thirdparty/faiss/faiss/gpu/GpuIndexIVFPQ.cu | 2 +- .../faiss/faiss/gpu/GpuIndexIVFSQHybrid.cu | 4 +- .../faiss/gpu/GpuIndexIVFScalarQuantizer.cu | 2 +- 12 files changed, 635 insertions(+), 301 deletions(-) create mode 100644 knowhere/utils/Bitset.cpp create mode 100644 knowhere/utils/Bitset.h create mode 100644 knowhere/utils/BitsetView.cpp delete mode 100644 thirdparty/faiss/Makefile.tmp diff --git a/knowhere/CMakeLists.txt b/knowhere/CMakeLists.txt index b4ea3849f..21dd0f74c 100644 --- a/knowhere/CMakeLists.txt +++ b/knowhere/CMakeLists.txt @@ -128,7 +128,6 @@ if ( LINUX ) set(depend_libs faiss pthread - knowhere_utils ) if (KNOWHERE_SUPPORT_SPTAG) @@ -179,10 +178,12 @@ if ( LINUX ) ${vector_index_srcs} ${vector_offset_index_srcs} ) - target_include_directories(knowhere PUBLIC ${KNOWHERE_SOURCE_DIR}/knowere) + target_include_directories(knowhere PUBLIC ${KNOWHERE_SOURCE_DIR}/knowhere) endif () - target_link_libraries(knowhere ${depend_libs}) + target_link_libraries(knowhere -Wl,--whole-archive knowhere_utils -Wl,--no-whole-archive) + #target_link_libraries(knowhere knowhere_utils) + set(KNOWHERE_INCLUDE_DIRS ${KNOWHERE_SOURCE_DIR} ${KNOWHERE_SOURCE_DIR}/thirdparty @@ -212,7 +213,10 @@ if (MACOS) ) endif () - target_link_libraries(knowhere pthread knowhere_utils) + target_link_libraries(knowhere pthread) + target_link_libraries(knowhere -Wl,--whole-archive knowhere_utils -Wl,--no-whole-archive) + #target_link_libraries(knowhere knowhere_utils) + set(KNOWHERE_INCLUDE_DIRS ${KNOWHERE_SOURCE_DIR}) endif() diff --git a/knowhere/index/vector_index/Statistics.h b/knowhere/index/vector_index/Statistics.h index af089be56..243456b0a 100644 --- a/knowhere/index/vector_index/Statistics.h +++ b/knowhere/index/vector_index/Statistics.h @@ -147,7 +147,7 @@ class Statistics { void update_filter_percentage(const faiss::BitsetView bitset) { - double fps = !bitset.empty() ? static_cast(bitset.count_1()) / bitset.size() : 0.0; + double fps = !bitset.empty() ? static_cast(bitset.count()) / bitset.size() : 0.0; filter_stat[static_cast(fps * 100) / 5] += 1; } diff --git a/knowhere/utils/Bitset.cpp b/knowhere/utils/Bitset.cpp new file mode 100644 index 000000000..76c2bc1b9 --- /dev/null +++ b/knowhere/utils/Bitset.cpp @@ -0,0 +1,327 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include +#include +#include +#include "Bitset.h" +#include "BitsetView.h" + +namespace faiss { + +ConcurrentBitset& +ConcurrentBitset::operator&=(const ConcurrentBitset& bitset) { + auto u8_1 = mutable_data(); + auto u8_2 = bitset.data(); + auto u64_1 = reinterpret_cast(u8_1); + auto u64_2 = reinterpret_cast(u8_2); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + u64_1[i] &= u64_2[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + u8_2 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + u8_1[i] &= u8_2[i]; + } + + return *this; +} + +ConcurrentBitset& +ConcurrentBitset::operator&=(const BitsetView& view) { + auto u8_1 = mutable_data(); + auto u8_2 = view.data(); + auto u64_1 = reinterpret_cast(u8_1); + auto u64_2 = reinterpret_cast(u8_2); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + u64_1[i] &= u64_2[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + u8_2 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + u8_1[i] &= u8_2[i]; + } + + return *this; +} + +std::shared_ptr +ConcurrentBitset::operator&(const ConcurrentBitset& bitset) const { +auto result_bitset = std::make_shared(bitset.count()); + +auto result_8 = result_bitset->mutable_data(); +auto result_64 = reinterpret_cast(result_8); + +auto u8_1 = data(); +auto u8_2 = bitset.data(); +auto u64_1 = reinterpret_cast(u8_1); +auto u64_2 = reinterpret_cast(u8_2); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + result_64[i] = u64_1[i] & u64_2[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +u8_2 += n64 * 8; +result_8 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + result_8[i] = u8_1[i] & u8_2[i]; +} + +return result_bitset; +} + +std::shared_ptr +ConcurrentBitset::operator&(const BitsetView& view) const { +auto result_bitset = std::make_shared(view.count()); + +auto result_8 = result_bitset->mutable_data(); +auto result_64 = reinterpret_cast(result_8); + +auto u8_1 = data(); +auto u8_2 = view.data(); +auto u64_1 = reinterpret_cast(u8_1); +auto u64_2 = reinterpret_cast(u8_2); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + result_64[i] = u64_1[i] & u64_2[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +u8_2 += n64 * 8; +result_8 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + result_8[i] = u8_1[i] & u8_2[i]; +} + +return result_bitset; +} + + +ConcurrentBitset& +ConcurrentBitset::operator|=(const ConcurrentBitset& bitset) { +auto u8_1 = mutable_data(); +auto u8_2 = bitset.data(); +auto u64_1 = reinterpret_cast(u8_1); +auto u64_2 = reinterpret_cast(u8_2); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + u64_1[i] |= u64_2[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +u8_2 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + u8_1[i] |= u8_2[i]; +} + +return *this; +} + +ConcurrentBitset& +ConcurrentBitset::operator|=(const BitsetView& view) { +auto u8_1 = mutable_data(); +auto u8_2 = view.data(); +auto u64_1 = reinterpret_cast(u8_1); +auto u64_2 = reinterpret_cast(u8_2); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + u64_1[i] |= u64_2[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +u8_2 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + u8_1[i] |= u8_2[i]; +} + +return *this; +} + + +std::shared_ptr +ConcurrentBitset::operator|(const ConcurrentBitset& bitset) const { +auto result_bitset = std::make_shared(bitset.count()); + +auto result_8 = result_bitset->mutable_data(); +auto result_64 = reinterpret_cast(result_8); + +auto u8_1 = data(); +auto u8_2 = bitset.data(); +auto u64_1 = reinterpret_cast(u8_1); +auto u64_2 = reinterpret_cast(u8_2); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + result_64[i] = u64_1[i] | u64_2[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +u8_2 += n64 * 8; +result_8 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + result_8[i] = u8_1[i] | u8_2[i]; +} + +return result_bitset; +} + +std::shared_ptr +ConcurrentBitset::operator|(const BitsetView& view) const { +auto result_bitset = std::make_shared(view.count()); + +auto result_8 = result_bitset->mutable_data(); +auto result_64 = reinterpret_cast(result_8); + +auto u8_1 = data(); +auto u8_2 = view.data(); +auto u64_1 = reinterpret_cast(u8_1); +auto u64_2 = reinterpret_cast(u8_2); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + result_64[i] = u64_1[i] | u64_2[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +u8_2 += n64 * 8; +result_8 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + result_8[i] = u8_1[i] | u8_2[i]; +} + +return result_bitset; +} + + +ConcurrentBitset& +ConcurrentBitset::negate() { +auto u8_1 = mutable_data(); +auto u64_1 = reinterpret_cast(u8_1); + +size_t n8 = bitset_.size(); +size_t n64 = n8 / 8; + +for (size_t i = 0; i < n64; i++) { + u64_1[i] = ~u64_1[i]; +} + +size_t remain = n8 % 8; +u8_1 += n64 * 8; +for (size_t i = 0; i < remain; i++) { + u8_1[i] = ~u8_1[i]; +} + +return *this; +} + +size_t +ConcurrentBitset::count() const { +size_t ret = 0; +auto p_data = reinterpret_cast(data()); +auto len = size() >> 3; +//auto remainder = size() % 8; +auto popcount8 = [&](uint8_t x) -> int{ + x = (x & 0x55) + ((x >> 1) & 0x55); + x = (x & 0x33) + ((x >> 2) & 0x33); + x = (x & 0x0F) + ((x >> 4) & 0x0F); + return x; +}; +for (size_t i = 0; i < len; ++i) { + ret += __builtin_popcountl(*p_data); + p_data++; +} +auto p_byte = data() + (len << 3); +for (auto i = (len << 3); i < size(); ++i) { + ret += popcount8(*p_byte); + p_byte++; +} +return ret; +} + +ConcurrentBitset::operator std::string() const { + const char one = '1'; + const char zero = '0'; + const size_t len = size(); + std::string s; + s.assign (len, zero); + + for (size_t i = 0; i < len; ++i) { + if (test(id_type_t(i))) + s.assign(len - 1 - i, one); + } + return s; +} + +bool operator==(const ConcurrentBitset& lhs, const ConcurrentBitset& rhs) { + if (std::addressof(lhs) == std::addressof(rhs)){ + return true; + } + + if (lhs.size() != rhs.size()){ + return false; + } + + if (lhs.byte_size() != rhs.byte_size()){ + return false; + } + + + auto ret = std::memcmp(lhs.data(), rhs.data(), lhs.byte_size()); + return ret == 0; +} + +bool operator!=(const ConcurrentBitset& lhs, const ConcurrentBitset& rhs){ + return !(lhs == rhs); +} + +std::ostream& operator<<(std::ostream& os, const ConcurrentBitset& bitset) +{ + os << std::string(bitset); + return os; +} + + +} // namespace faiss diff --git a/knowhere/utils/Bitset.h b/knowhere/utils/Bitset.h new file mode 100644 index 000000000..460c28dfc --- /dev/null +++ b/knowhere/utils/Bitset.h @@ -0,0 +1,131 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace faiss { + +class BitsetView; + +class ConcurrentBitset { + using id_type_t = int64_t; + + friend + bool operator==(const ConcurrentBitset& lhs, const ConcurrentBitset& rhs); + + friend + bool operator!=(const ConcurrentBitset& lhs, const ConcurrentBitset& rhs); + + friend + std::ostream& operator<<(std::ostream& os, const ConcurrentBitset& bitset); + + public: + + explicit ConcurrentBitset(size_t size, uint8_t init_value = 0) + : size_(size), bitset_(((size + 8 - 1) >> 3)) { + if (init_value) { + memset(mutable_data(), init_value, (size_ + 8 - 1) >> 3); + } + } + + explicit ConcurrentBitset(size_t size, const uint8_t* data) : size_(size), bitset_(((size + 8 - 1) >> 3)) { + memcpy(mutable_data(), data, (size_ + 8 - 1) >> 3); + } + + ConcurrentBitset& + operator&=(const ConcurrentBitset& bitset); + + ConcurrentBitset& + operator&=(const BitsetView& view); + + std::shared_ptr + operator&(const ConcurrentBitset& bitset) const; + + std::shared_ptr + operator&(const BitsetView& view) const; + + ConcurrentBitset& + operator|=(const ConcurrentBitset& bitset); + + ConcurrentBitset& + operator|=(const BitsetView& view); + + std::shared_ptr + operator|(const ConcurrentBitset& bitset) const; + + std::shared_ptr + operator|(const BitsetView& view) const; + + ConcurrentBitset& + negate(); + + inline bool + test(id_type_t id) const { + unsigned char mask = (unsigned char)(0x01) << (id & 0x07); + return (bitset_[id >> 3].load() & mask); + } + + inline void + set(id_type_t id) { + unsigned char mask = (unsigned char)(0x01) << (id & 0x07); + bitset_[id >> 3].fetch_or(mask); + } + + inline void + clear(id_type_t id) { + unsigned char mask = (unsigned char)(0x01) << (id & 0x07); + bitset_[id >> 3].fetch_and(~mask); + } + + size_t + count() const ; + + inline size_t + size() const { + return size_; + } + + inline size_t + byte_size() const { + return ((size_ + 8 - 1) >> 3); + } + + inline const uint8_t* + data() const { + return reinterpret_cast(bitset_.data()); + } + + inline uint8_t* + mutable_data() { + return reinterpret_cast(bitset_.data()); + } + + operator std::string() const; + + private: + size_t size_; // number of bits + std::vector> bitset_; +}; + +bool operator==(const ConcurrentBitset& lhs, const ConcurrentBitset& rhs); +bool operator!=(const ConcurrentBitset& lhs, const ConcurrentBitset& rhs); +std::ostream& operator<<(std::ostream& os, const ConcurrentBitset& bitset); + +using ConcurrentBitsetPtr = std::shared_ptr; + +} // namespace faiss diff --git a/knowhere/utils/BitsetView.cpp b/knowhere/utils/BitsetView.cpp new file mode 100644 index 000000000..7dc8e7419 --- /dev/null +++ b/knowhere/utils/BitsetView.cpp @@ -0,0 +1,128 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include +#include +#include +#include +#include +#include "BitsetView.h" + +namespace faiss { + + bool + BitsetView::empty() const { + return size_ == 0; + } + + // return count of all bits + size_t + BitsetView::size() const { + return size_; + } + + + // return sizeof bitmap in bytes + size_t + BitsetView::byte_size() const { + return (size_ + 8 - 1) >> 3; + } + + const uint8_t* + BitsetView::data() const { + return blocks_; + } + + uint8_t* + BitsetView::mutable_data() { + return const_cast(blocks_); + } + + bool + BitsetView::test(int64_t index) const { + auto block_id = index >> 3; + auto block_offset = index & 0x7; + return (blocks_[block_id] >> block_offset) & 0x1; + } + + BitsetView::operator bool() const { + return !empty(); + } + + size_t + BitsetView::count() const { + size_t ret = 0; + auto p_data = reinterpret_cast(blocks_); + auto len = size_ >> 6; + //auto remainder = size() % 8; + auto popcount8 = [&](uint8_t x) -> int{ + x = (x & 0x55) + ((x >> 1) & 0x55); + x = (x & 0x33) + ((x >> 2) & 0x33); + x = (x & 0x0F) + ((x >> 4) & 0x0F); + return x; + }; + for (int64_t i = 0; i < len; ++i) { + ret += __builtin_popcountl(*p_data); + p_data++; + } + auto p_byte = blocks_ + (len << 3); + for (auto i = (len << 3); i < byte_size(); ++i) { + ret += popcount8(*p_byte); + p_byte++; + } + return ret; + } + + +BitsetView::operator std::string() const { + const char one = '1'; + const char zero = '0'; + const size_t len = size(); + std::string s; + s.assign (len, zero); + + for (size_t i = 0; i < len; ++i) { + if (test(i)) + s.assign(len - 1 - i, one); + } + return s; +} + +bool operator==(const BitsetView& lhs, const BitsetView& rhs) { + if (std::addressof(lhs) == std::addressof(rhs)){ + return true; + } + + if (lhs.size() != rhs.size()){ + return false; + } + + if (lhs.byte_size() != rhs.byte_size()){ + return false; + } + + + auto ret = std::memcmp(lhs.data(), rhs.data(), lhs.byte_size()); + return ret == 0; +} + +bool operator!=(const BitsetView& lhs, const BitsetView& rhs){ + return !(lhs == rhs); +} + +std::ostream& operator<<(std::ostream& os, const BitsetView& bitset) +{ + os << std::string(bitset); + return os; +} + + +} // namespace faiss diff --git a/knowhere/utils/BitsetView.h b/knowhere/utils/BitsetView.h index 88ecd7d4c..60b2616fa 100644 --- a/knowhere/utils/BitsetView.h +++ b/knowhere/utils/BitsetView.h @@ -12,238 +12,34 @@ #pragma once #include +#include #include #include #include #include -namespace faiss { - -class ConcurrentBitset { - public: - using id_type_t = int64_t; - - explicit ConcurrentBitset(size_t count, uint8_t init_value = 0) - : count_(count), bitset_(((count + 8 - 1) >> 3)) { - if (init_value) { - memset(mutable_data(), init_value, (count + 8 - 1) >> 3); - } - } - - explicit ConcurrentBitset(size_t count, const uint8_t* data) : count_(count), bitset_(((count + 8 - 1) >> 3)) { - memcpy(mutable_data(), data, (count + 8 - 1) >> 3); - } - - ConcurrentBitset& - operator&=(const ConcurrentBitset& bitset) { - auto u8_1 = mutable_data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] &= u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] &= u8_2[i]; - } - - return *this; - } - - std::shared_ptr - operator&(const ConcurrentBitset& bitset) const { - auto result_bitset = std::make_shared(bitset.count()); - - auto result_8 = result_bitset->mutable_data(); - auto result_64 = reinterpret_cast(result_8); - - auto u8_1 = data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - result_64[i] = u64_1[i] & u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - result_8 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - result_8[i] = u8_1[i] & u8_2[i]; - } - - return result_bitset; - } - - ConcurrentBitset& - operator|=(const ConcurrentBitset& bitset) { - auto u8_1 = mutable_data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] |= u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] |= u8_2[i]; - } - - return *this; - } - - std::shared_ptr - operator|(const ConcurrentBitset& bitset) const { - auto result_bitset = std::make_shared(bitset.count()); - - auto result_8 = result_bitset->mutable_data(); - auto result_64 = reinterpret_cast(result_8); - - auto u8_1 = data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - result_64[i] = u64_1[i] | u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - result_8 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - result_8[i] = u8_1[i] | u8_2[i]; - } - - return result_bitset; - } - - ConcurrentBitset& - negate() { - auto u8_1 = mutable_data(); - auto u64_1 = reinterpret_cast(u8_1); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] = ~u64_1[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] = ~u8_1[i]; - } - - return *this; - } - - bool - test(id_type_t id) { - unsigned char mask = (unsigned char)(0x01) << (id & 0x07); - return (bitset_[id >> 3].load() & mask); - } - - void - set(id_type_t id) { - unsigned char mask = (unsigned char)(0x01) << (id & 0x07); - bitset_[id >> 3].fetch_or(mask); - } +#include "Bitset.h" - void - clear(id_type_t id) { - unsigned char mask = (unsigned char)(0x01) << (id & 0x07); - bitset_[id >> 3].fetch_and(~mask); - } +namespace faiss { - size_t - count() const { - return count_; - } +class BitsetView { - size_t - size() const { - return ((count_ + 8 - 1) >> 3); - } + friend + bool operator==(const BitsetView& lhs, const BitsetView& rhs); - const uint8_t* - data() const { - return reinterpret_cast(bitset_.data()); - } + friend + bool operator!=(const BitsetView& lhs, const BitsetView& rhs); - uint8_t* - mutable_data() { - return reinterpret_cast(bitset_.data()); - } + friend + std::ostream& operator<<(std::ostream& os, const BitsetView& view); - uint64_t - count_1() { - uint64_t ret = 0; - auto p_data = reinterpret_cast(mutable_data()); - auto len = size() >> 3; - //auto remainder = size() % 8; - auto popcount8 = [&](uint8_t x) -> int{ - x = (x & 0x55) + ((x >> 1) & 0x55); - x = (x & 0x33) + ((x >> 2) & 0x33); - x = (x & 0x0F) + ((x >> 4) & 0x0F); - return x; - }; - for (size_t i = 0; i < len; ++i) { - ret += __builtin_popcountl(*p_data); - p_data++; - } - auto p_byte = data() + (len << 3); - for (auto i = (len << 3); i < size(); ++i) { - ret += popcount8(*p_byte); - p_byte++; - } - return ret; - } - - private: - size_t count_; - std::vector> bitset_; -}; -using ConcurrentBitsetPtr = std::shared_ptr; - - -class BitsetView { public: BitsetView() = default; BitsetView(const uint8_t* blocks, int64_t size) : blocks_(blocks), size_(size) { } - explicit BitsetView(const ConcurrentBitset& bitset) : size_(bitset.count()) { - // memcpy(block_data_.data(), bitset.data(), bitset.size()); - // blocks_ = block_data_.data(); - blocks_ = new uint8_t[bitset.size()]; - memcpy(mutable_data(), bitset.data(), bitset.size()); + explicit BitsetView(const ConcurrentBitset& bitset) : blocks_(bitset.data()), size_(bitset.size()) { } BitsetView(const ConcurrentBitsetPtr& bitset_ptr) { @@ -257,71 +53,40 @@ class BitsetView { } bool - empty() const { - return size_ == 0; - } + empty() const; + + // return count of all 1-bits + size_t + count() const; // return count of all bits - int64_t - size() const { - return size_; - } + size_t + size() const; // return sizeof bitmap in bytes - int64_t - u8size() const { - return (size_ + 8 - 1) >> 3; - } + size_t + byte_size() const; const uint8_t* - data() const { - return blocks_; - } + data() const; uint8_t* - mutable_data() { - return const_cast(blocks_); - } - - operator bool() const { - return !empty(); - } + mutable_data(); bool - test(int64_t index) const { - // assert(index < size_); - auto block_id = index >> 3; - auto block_offset = index & 0x7; - return (blocks_[block_id] >> block_offset) & 0x1; - } + test(int64_t index) const; - uint64_t - count_1() const { - uint64_t ret = 0; - auto p_data = reinterpret_cast(blocks_); - auto len = size_ >> 6; - //auto remainder = size() % 8; - auto popcount8 = [&](uint8_t x) -> int{ - x = (x & 0x55) + ((x >> 1) & 0x55); - x = (x & 0x33) + ((x >> 2) & 0x33); - x = (x & 0x0F) + ((x >> 4) & 0x0F); - return x; - }; - for (int64_t i = 0; i < len; ++i) { - ret += __builtin_popcountl(*p_data); - p_data++; - } - auto p_byte = blocks_ + (len << 3); - for (auto i = (len << 3); i < u8size(); ++i) { - ret += popcount8(*p_byte); - p_byte++; - } - return ret; - } + operator bool() const; + operator std::string() const; private: const uint8_t* blocks_ = nullptr; - int64_t size_ = 0; // size of bits + size_t size_ = 0; // count of bits }; +bool operator==(const BitsetView& lhs, const BitsetView& rhs); +bool operator!=(const BitsetView& lhs, const BitsetView& rhs); +std::ostream& operator<<(std::ostream& os, const BitsetView& view); + + } // namespace faiss diff --git a/knowhere/utils/CMakeLists.txt b/knowhere/utils/CMakeLists.txt index 0c5b42aff..1428a587d 100644 --- a/knowhere/utils/CMakeLists.txt +++ b/knowhere/utils/CMakeLists.txt @@ -17,12 +17,16 @@ if (MACOS) set(UTILS_SRC distances_simd.cpp + Bitset.cpp + BitsetView.cpp ) add_library(knowhere_utils STATIC ${UTILS_SRC} ) else () set(UTILS_SRC + BitsetView.cpp + Bitset.cpp distances_simd.cpp FaissHookFvec.cpp ) @@ -49,7 +53,6 @@ else () target_compile_options(utils_sse PUBLIC "-msse4.2") target_compile_options(utils_avx PUBLIC "-mf16c;-mavx2") target_compile_options(utils_avx512 PUBLIC "-mf16c;-mavx512f;-mavx512dq;-mavx512bw") - add_library(knowhere_utils STATIC ${UTILS_SRC} $ diff --git a/thirdparty/faiss/Makefile.tmp b/thirdparty/faiss/Makefile.tmp deleted file mode 100644 index 963974426..000000000 --- a/thirdparty/faiss/Makefile.tmp +++ /dev/null @@ -1,24 +0,0 @@ -cpu: - cmake -B cmake_build . \ - -DCMAKE_BUILD_TYPE=Release \ - -DFAISS_ENABLE_PYTHON=OFF \ - -DFAISS_ENABLE_GPU=OFF \ - -DBUILD_TESTING=ON - make -C cmake_build -j8 - -gpu: - cmake -B cmake_build . \ - -DCMAKE_BUILD_TYPE=Release \ - -DFAISS_ENABLE_PYTHON=OFF \ - -DFAISS_ENABLE_GPU=ON \ - -DCUDAToolkit_ROOT=/usr/local/cuda \ - -DCMAKE_CUDA_ARCHITECTURES="75" \ - -DBUILD_TESTING=ON - make -C cmake_build -j8 - -test: - make -C cmake_build test - -clean: - rm -rf cmake_build - diff --git a/thirdparty/faiss/faiss/gpu/GpuIndexIVFFlat.cu b/thirdparty/faiss/faiss/gpu/GpuIndexIVFFlat.cu index 695187f90..ac214d8e7 100644 --- a/thirdparty/faiss/faiss/gpu/GpuIndexIVFFlat.cu +++ b/thirdparty/faiss/faiss/gpu/GpuIndexIVFFlat.cu @@ -346,7 +346,7 @@ void GpuIndexIVFFlat::searchImpl_( config_.device, const_cast(bitset.data()), stream, - {(int)bitset.u8size()}); + {(int)bitset.byte_size()}); index_->query( queries, bitsetDevice, diff --git a/thirdparty/faiss/faiss/gpu/GpuIndexIVFPQ.cu b/thirdparty/faiss/faiss/gpu/GpuIndexIVFPQ.cu index d57bb1624..acdd7e61a 100644 --- a/thirdparty/faiss/faiss/gpu/GpuIndexIVFPQ.cu +++ b/thirdparty/faiss/faiss/gpu/GpuIndexIVFPQ.cu @@ -387,7 +387,7 @@ void GpuIndexIVFPQ::searchImpl_( config_.device, const_cast(bitset.data()), stream, - {(int)bitset.u8size()}); + {(int)bitset.byte_size()}); index_->query( queries, bitsetDevice, diff --git a/thirdparty/faiss/faiss/gpu/GpuIndexIVFSQHybrid.cu b/thirdparty/faiss/faiss/gpu/GpuIndexIVFSQHybrid.cu index c1964d81b..6cfe5d4cd 100644 --- a/thirdparty/faiss/faiss/gpu/GpuIndexIVFSQHybrid.cu +++ b/thirdparty/faiss/faiss/gpu/GpuIndexIVFSQHybrid.cu @@ -339,7 +339,7 @@ void GpuIndexIVFSQHybrid::searchImpl_( config_.device, const_cast(bitset.data()), stream, - {(int)bitset.u8size()}); + {(int)bitset.byte_size()}); index_->query( queries, bitsetDevice, @@ -351,4 +351,4 @@ void GpuIndexIVFSQHybrid::searchImpl_( } } // namespace gpu -} // namespace faiss \ No newline at end of file +} // namespace faiss diff --git a/thirdparty/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.cu b/thirdparty/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.cu index 16ad502ec..fe1deca0d 100644 --- a/thirdparty/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.cu +++ b/thirdparty/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.cu @@ -369,7 +369,7 @@ void GpuIndexIVFScalarQuantizer::searchImpl_( config_.device, const_cast(bitset.data()), stream, - {(int)bitset.u8size()}); + {(int)bitset.byte_size()}); index_->query(queries, bitsetDevice, nprobe, k, outDistances, outLabels); } }