Skip to content

Commit

Permalink
add compaction service
Browse files Browse the repository at this point in the history
  • Loading branch information
Qiaolin-Yu committed Mar 15, 2024
1 parent d6c3537 commit ef083f2
Show file tree
Hide file tree
Showing 29 changed files with 2,296 additions and 170 deletions.
79 changes: 76 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,9 @@ endif(CCACHE_FOUND)

option(WITH_JEMALLOC "build with JeMalloc" OFF)
option(WITH_LIBURING "build with liburing" ON)
option(WITH_SNAPPY "build with SNAPPY" OFF)
option(WITH_LZ4 "build with lz4" OFF)
option(WITH_SNAPPY "build with SNAPPY" ON)
option(WITH_HDFS "build with HDFS and write data to HDFS" OFF)
option(WITH_LZ4 "build with lz4" ON)
option(WITH_ZLIB "build with zlib" OFF)
option(WITH_ZSTD "build with zstd" OFF)
option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
Expand All @@ -87,6 +88,8 @@ endif()

include(CMakeDependentOption)

include(${PROJECT_SOURCE_DIR}/cmake/modules/grpc.cmake)

if(MSVC)
option(WITH_GFLAGS "build with GFlags" OFF)
option(WITH_XPRESS "build with windows built in compression" OFF)
Expand Down Expand Up @@ -643,6 +646,49 @@ if(USE_FOLLY)
endif()
find_package(Threads REQUIRED)

# Proto file
get_filename_component(compaction_service_proto "db/compaction/remote_compaction/protos/compaction_service.proto" ABSOLUTE)
get_filename_component(compaction_service_proto_path "${compaction_service_proto}" PATH)

# Generated sources
set(compaction_service_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/compaction_service.pb.cc")
set(compaction_service_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/compaction_service.pb.h")
set(compaction_service_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/compaction_service.grpc.pb.cc")
set(compaction_service_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/compaction_service.grpc.pb.h")
add_custom_command(
OUTPUT "${compaction_service_proto_srcs}" "${compaction_service_proto_hdrs}" "${compaction_service_grpc_srcs}" "${compaction_service_grpc_hdrs}"
COMMAND ${_PROTOBUF_PROTOC}
ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
--cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
-I "${compaction_service_proto_path}"
--plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
"${compaction_service_proto}"
DEPENDS "${compaction_service_proto}")

# Include generated *.pb.h files
include_directories("${CMAKE_CURRENT_BINARY_DIR}")

# compaction_service_grpc_proto
add_library(compaction_service_grpc_proto
${compaction_service_grpc_srcs}
${compaction_service_grpc_hdrs}
${compaction_service_proto_srcs}
${compaction_service_proto_hdrs})
target_link_libraries(compaction_service_grpc_proto
${_REFLECTION}
${_GRPC_GRPCPP}
${_PROTOBUF_LIBPROTOBUF})

# compaction_service_helper
#add_library(compaction_service_helper
# "db/compaction/remote_compaction/job_distributor.h"
# "db/compaction/remote_compaction/job_distributor.cc")
#target_link_libraries(compaction_service_helper
# compaction_service_grpc_proto
# ${_REFLECTION}
# ${_GRPC_GRPCPP}
# ${_PROTOBUF_LIBPROTOBUF})

# Main library source code

set(SOURCES
Expand Down Expand Up @@ -952,7 +998,11 @@ set(SOURCES
utilities/ttl/db_ttl_impl.cc
utilities/wal_filter.cc
utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc)
utilities/write_batch_with_index/write_batch_with_index_internal.cc
db/compaction/compaction_service.cc
${compaction_service_grpc_srcs}
${compaction_service_proto_srcs}
)

list(APPEND SOURCES
utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
Expand All @@ -968,6 +1018,11 @@ list(APPEND SOURCES
utilities/transactions/lock/range/range_tree/lib/util/dbt.cc
utilities/transactions/lock/range/range_tree/lib/util/memarena.cc)

if(WITH_HDFS)
set(ROCKSDB_PLUGINS "hdfs")
add_definitions(-DHDFS)
endif ()

message(STATUS "ROCKSDB_PLUGINS: ${ROCKSDB_PLUGINS}")
if ( ROCKSDB_PLUGINS )
string(REPLACE " " ";" PLUGINS ${ROCKSDB_PLUGINS})
Expand Down Expand Up @@ -1051,6 +1106,11 @@ if(USE_FOLLY_LITE)
list(APPEND THIRDPARTY_LIBS glog)
endif()

list(APPEND THIRDPARTY_LIBS
${_REFLECTION}
${_GRPC_GRPCPP}
${_PROTOBUF_LIBPROTOBUF})

set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})

Expand Down Expand Up @@ -1486,6 +1546,19 @@ if(WITH_TESTS)
set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB} ${FOLLY_LIBS})

# Targets compaction_service_(client|server)
foreach(_target csa_server procp_server)
add_executable(${_target}
"db/compaction/remote_compaction/${_target}.cc")
target_link_libraries(${_target}
# compaction_service_helper
${_REFLECTION}
${_GRPC_GRPCPP}
${_PROTOBUF_LIBPROTOBUF}
${ROCKSDB_LIB})
endforeach()

if(MSVC)
set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
endif()
Expand Down
128 changes: 128 additions & 0 deletions cmake/modules/grpc.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
# Copyright 2018 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# cmake build file for C++ route_guide example.
# Assumes protobuf and gRPC have been installed using cmake.
# See cmake_externalproject/CMakeLists.txt for all-in-one cmake build
# that automatically builds all the dependencies before building route_guide.

cmake_minimum_required(VERSION 3.5.1)

if (NOT DEFINED CMAKE_CXX_STANDARD)
set (CMAKE_CXX_STANDARD 14)
endif()

if(MSVC)
add_definitions(-D_WIN32_WINNT=0x600)
endif()

find_package(Threads REQUIRED)

if(GRPC_AS_SUBMODULE)
# One way to build a projects that uses gRPC is to just include the
# entire gRPC project tree via "add_subdirectory".
# This approach is very simple to use, but the are some potential
# disadvantages:
# * it includes gRPC's CMakeLists.txt directly into your build script
# without and that can make gRPC's internal setting interfere with your
# own build.
# * depending on what's installed on your system, the contents of submodules
# in gRPC's third_party/* might need to be available (and there might be
# additional prerequisites required to build them). Consider using
# the gRPC_*_PROVIDER options to fine-tune the expected behavior.
#
# A more robust approach to add dependency on gRPC is using
# cmake's ExternalProject_Add (see cmake_externalproject/CMakeLists.txt).

# Include the gRPC's cmake build (normally grpc source code would live
# in a git submodule called "third_party/grpc", but this example lives in
# the same repository as gRPC sources, so we just look a few directories up)
add_subdirectory(. ${CMAKE_CURRENT_BINARY_DIR}/grpc EXCLUDE_FROM_ALL)
message(STATUS "Using gRPC via add_subdirectory.")

# After using add_subdirectory, we can now use the grpc targets directly from
# this build.
set(_PROTOBUF_LIBPROTOBUF libprotobuf)
set(_REFLECTION grpc++_reflection)
if(CMAKE_CROSSCOMPILING)
find_program(_PROTOBUF_PROTOC protoc)
else()
set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
endif()
set(_GRPC_GRPCPP grpc++)
if(CMAKE_CROSSCOMPILING)
find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
else()
set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
endif()
elseif(GRPC_FETCHCONTENT)
# Another way is to use CMake's FetchContent module to clone gRPC at
# configure time. This makes gRPC's source code available to your project,
# similar to a git submodule.
message(STATUS "Using gRPC via add_subdirectory (FetchContent).")
include(FetchContent)
FetchContent_Declare(
grpc
GIT_REPOSITORY https://github.com/grpc/grpc.git
# when using gRPC, you will actually set this to an existing tag, such as
# v1.25.0, v1.26.0 etc..
# For the purpose of testing, we override the tag used to the commit
# that's currently under test.
GIT_TAG v1.46.2)
FetchContent_MakeAvailable(grpc)

# Since FetchContent uses add_subdirectory under the hood, we can use
# the grpc targets directly from this build.
set(_PROTOBUF_LIBPROTOBUF libprotobuf)
set(_REFLECTION grpc++_reflection)
set(_PROTOBUF_PROTOC $<TARGET_FILE:protoc>)
set(_GRPC_GRPCPP grpc++)
if(CMAKE_CROSSCOMPILING)
find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
else()
set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
endif()
else()
# This branch assumes that gRPC and all its dependencies are already installed
# on this system, so they can be located by find_package().

# Find Protobuf installation
# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
set(protobuf_MODULE_COMPATIBLE TRUE)
find_package(Protobuf CONFIG)
if(NOT Protobuf_FOUND)
find_package(Protobuf REQUIRED)
endif()
message(STATUS "Using protobuf ${Protobuf_VERSION}")

set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
set(_REFLECTION gRPC::grpc++_reflection)
if(CMAKE_CROSSCOMPILING)
find_program(_PROTOBUF_PROTOC protoc)
else()
set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
endif()

# Find gRPC installation
# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
find_package(gRPC CONFIG REQUIRED)
message(STATUS "Using gRPC ${gRPC_VERSION}")

set(_GRPC_GRPCPP gRPC::grpc++)
if(CMAKE_CROSSCOMPILING)
find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
else()
set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
endif()
endif()
Loading

0 comments on commit ef083f2

Please sign in to comment.