Add cpp data codec (milvus-io#18538)

Signed-off-by: xige-16 <[email protected]>
Co-authored-by: zhagnlu [email protected]

Signed-off-by: xige-16 <[email protected]>

.gitignore

@@ -89,3 +89,6 @@ deployments/docker/*/volumes
 # rocksdb
 cwrapper_rocksdb_build/
 internal/kv/rocksdb/cwrapper/
+
+# local file data
+**/data/*

internal/core/src/common/Consts.h

@@ -25,3 +25,10 @@ const milvus::PkType INVALID_PK;  // of std::monostate if not set.
 // TODO: default field start id, could get from config.yaml
 const int64_t START_USER_FIELDID = 100;
 const char MAX_LENGTH[] = "max_length";
+
+// fill followed extra info to binlog file
+const char ORIGIN_SIZE_KEY[] = "original_size";
+const char INDEX_BUILD_ID_KEY[] = "indexBuildID";
+
+const char INDEX_ROOT_PATH[] = "index_files";
+const char RAWDATA_ROOT_PATH[] = "raw_datas";

internal/core/src/config/CMakeLists.txt

@@ -22,6 +22,7 @@ endif()
 
 set(CONFIG_SRC
         ConfigKnowhere.cpp
+        ConfigChunkManager.cpp
         )
 
 add_library(milvus_config STATIC ${CONFIG_SRC})

internal/core/src/config/ConfigChunkManager.cpp

@@ -0,0 +1,88 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "config/ConfigChunkManager.h"
+
+namespace milvus::ChunkMangerConfig {
+
+std::string MINIO_ADDRESS = "localhost:9000";   // NOLINT
+std::string MINIO_ACCESS_KEY = "minioadmin";    // NOLINT
+std::string MINIO_ACCESS_VALUE = "minioadmin";  // NOLINT
+std::string MINIO_BUCKET_NAME = "a-bucket";     // NOLINT
+std::string LOCAL_BUCKET_NAME = "/tmp/milvus";  // NOLINT
+bool MINIO_USE_SSL = false;
+
+void
+SetAddress(const std::string& address) {
+    MINIO_ADDRESS = address.c_str();
+}
+
+std::string
+GetAddress() {
+    return MINIO_ADDRESS;
+}
+
+void
+SetAccessKey(const std::string& access_key) {
+    MINIO_ACCESS_KEY = access_key.c_str();
+}
+
+std::string
+GetAccessKey() {
+    return MINIO_ACCESS_KEY;
+}
+
+void
+SetAccessValue(const std::string& access_value) {
+    MINIO_ACCESS_VALUE = access_value.c_str();
+}
+
+std::string
+GetAccessValue() {
+    return MINIO_ACCESS_VALUE;
+}
+
+void
+SetBucketName(const std::string& bucket_name) {
+    MINIO_BUCKET_NAME = bucket_name.c_str();
+}
+
+std::string
+GetBucketName() {
+    return MINIO_BUCKET_NAME;
+}
+
+void
+SetUseSSL(bool use_ssl) {
+    MINIO_USE_SSL = use_ssl;
+}
+
+bool
+GetUseSSL() {
+    return MINIO_USE_SSL;
+}
+
+void
+SetLocalBucketName(const std::string& path_prefix) {
+    LOCAL_BUCKET_NAME = path_prefix.c_str();
+}
+
+std::string
+GetLocalBucketName() {
+    return LOCAL_BUCKET_NAME;
+}
+
+}  // namespace milvus::ChunkMangerConfig

internal/core/src/config/ConfigChunkManager.h

@@ -0,0 +1,59 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+
+namespace milvus::ChunkMangerConfig {
+
+void
+SetAddress(const std::string& address);
+
+std::string
+GetAddress();
+
+void
+SetAccessKey(const std::string& access_key);
+
+std::string
+GetAccessKey();
+
+void
+SetAccessValue(const std::string& access_value);
+
+std::string
+GetAccessValue();
+
+void
+SetUseSSL(bool use_ssl);
+
+bool
+GetUseSSL();
+
+void
+SetBucketName(const std::string& bucket_name);
+
+std::string
+GetBucketName();
+
+void
+SetLocalBucketName(const std::string& path_prefix);
+
+std::string
+GetLocalBucketName();
+
+}  // namespace milvus::ChunkMangerConfig

internal/core/src/config/ConfigKnowhere.cpp

@@ -78,4 +78,9 @@ KnowhereSetIndexSliceSize(const int64_t size) {
     knowhere::KnowhereConfig::SetIndexFileSliceSize(size);
 }
 
+int64_t
+KnowhereGetIndexSliceSize() {
+    return knowhere::KnowhereConfig::GetIndexFileSliceSize();
+}
+
 }  // namespace milvus::config

internal/core/src/config/ConfigKnowhere.h

@@ -28,4 +28,7 @@ KnowhereSetSimdType(const char*);
 void
 KnowhereSetIndexSliceSize(const int64_t size);
 
+int64_t
+KnowhereGetIndexSliceSize();
+
 }  // namespace milvus::config

internal/core/src/storage/CMakeLists.txt

@@ -23,10 +23,21 @@ endif()
 
 milvus_add_pkg_config("milvus_storage")
 
-set(STORAGE_FILES parquet_c.cpp PayloadStream.cpp)
+set(STORAGE_FILES
+        parquet_c.cpp
+        PayloadStream.cpp
+        DataCodec.cpp
+        Util.cpp
+        PayloadReader.cpp
+        PayloadWriter.cpp
+        FieldData.cpp
+        IndexData.cpp
+        InsertData.cpp
+        Event.cpp
+        )
 add_library(milvus_storage SHARED ${STORAGE_FILES})
-
-target_link_libraries( milvus_storage PUBLIC arrow parquet pthread)
+#target_link_libraries( milvus_storage PUBLIC milvus_common boost_system boost_filesystem aws-cpp-sdk-s3 pthread)
+target_link_libraries( milvus_storage PUBLIC milvus_common pthread)
 
 if(NOT CMAKE_INSTALL_PREFIX)
     set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})

internal/core/src/storage/ChunkManager.h

@@ -0,0 +1,128 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace milvus::storage {
+
+/**
+ * @brief This ChunkManager is abstract interface for milvus that
+ * used to manager operation and interaction with storage
+ */
+class ChunkManager {
+ public:
+    /**
+     * @brief Whether file exists or not
+     * @param filepath
+     * @return true
+     * @return false
+     */
+    virtual bool
+    Exist(const std::string& filepath) = 0;
+
+    /**
+     * @brief Get file size
+     * @param filepath
+     * @return uint64_t
+     */
+    virtual uint64_t
+    Size(const std::string& filepath) = 0;
+
+    /**
+     * @brief Read file to buffer
+     * @param filepath
+     * @param buf
+     * @param len
+     * @return uint64_t
+     */
+    virtual uint64_t
+    Read(const std::string& filepath, void* buf, uint64_t len) = 0;
+
+    /**
+     * @brief Write buffer to file with offset
+     * @param filepath
+     * @param buf
+     * @param len
+     */
+    virtual void
+    Write(const std::string& filepath, void* buf, uint64_t len) = 0;
+
+    /**
+     * @brief Read file to buffer with offset
+     * @param filepath
+     * @param buf
+     * @param len
+     * @return uint64_t
+     */
+    virtual uint64_t
+    Read(const std::string& filepath, uint64_t offset, void* buf, uint64_t len) = 0;
+
+    /**
+     * @brief Write buffer to file with offset
+     * @param filepath
+     * @param buf
+     * @param len
+     */
+    virtual void
+    Write(const std::string& filepath, uint64_t offset, void* buf, uint64_t len) = 0;
+
+    /**
+     * @brief List files with same prefix
+     * @param filepath
+     * @return std::vector<std::string>
+     */
+    virtual std::vector<std::string>
+    ListWithPrefix(const std::string& filepath) = 0;
+
+    /**
+     * @brief Remove specified file
+     * @param filepath
+     */
+    virtual void
+    Remove(const std::string& filepath) = 0;
+
+    /**
+     * @brief Get the Name object
+     * Used for forming diagnosis messages
+     * @return std::string
+     */
+    virtual std::string
+    GetName() const = 0;
+};
+
+/**
+ * @brief RemoteChunkManager is responsible for read and write Remote file
+ * that inherited from ChunkManager.
+ */
+
+class RemoteChunkManager : public ChunkManager {
+ public:
+    virtual ~RemoteChunkManager() {
+    }
+    virtual std::string
+    GetName() const {
+        return "RemoteChunkManager";
+    }
+};
+
+using RemoteChunkManagerSPtr = std::shared_ptr<milvus::storage::RemoteChunkManager>;
+
+}  // namespace milvus::storage