diff --git a/CMakeLists.txt b/CMakeLists.txt index 138a0ca..a312719 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE) string(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -g -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ") -string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra ") +string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra") ### # Mandatory: devices configuration file. @@ -127,6 +127,7 @@ target_link_libraries(rdmalib PRIVATE cereal) # client library ### file(GLOB rdmalib_files "rfaas/lib/*.cpp") +message(STATUS "rdmalib_files ${rdmalib_files}") add_library(rfaaslib STATIC ${rdmalib_files}) add_dependencies(rfaaslib spdlog) add_dependencies(rfaaslib cereal) diff --git a/benchmarks/warm_benchmark.cpp b/benchmarks/warm_benchmark.cpp index d16f36b..ad45b6d 100644 --- a/benchmarks/warm_benchmark.cpp +++ b/benchmarks/warm_benchmark.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "warm_benchmark.hpp" #include "settings.hpp" @@ -68,19 +69,27 @@ int main(int argc, char ** argv) } // FIXME: move me to a memory allocator - rdmalib::Buffer in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size); - in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE); - out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - memset(in.data(), 0, opts.input_size); - for(int i = 0; i < opts.input_size; ++i) { - ((char*)in.data())[i] = 1; + + rfaas::RdmaAllocator > rdmaAllocator(executor); + rdmalib::Buffer* in = rdmaAllocator.allocate(opts.input_size, IBV_ACCESS_LOCAL_WRITE, + rdmalib::functions::Submission::DATA_HEADER_SIZE); + rdmalib::Buffer* out = rdmaAllocator.allocate(opts.input_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + // rdmalib::Buffer in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size); + // in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE); + // out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + + // TODO: Since the for loop writes a value of 1 to each byte of the in buffer, + // it overwrites all bytes previously set to 0 by the memset() function. + memset(in->data(), 0, opts.input_size); + for (int i = 0; i < opts.input_size; ++i) { + ((char *) in->data())[i] = 1; } rdmalib::Benchmarker<1> benchmarker{settings.benchmark.repetitions}; spdlog::info("Warmups begin"); for(int i = 0; i < settings.benchmark.warmup_repetitions; ++i) { SPDLOG_DEBUG("Submit warm {}", i); - executor.execute(opts.fname, in, out); + executor.execute(opts.fname, *in, *out); } spdlog::info("Warmups completed"); @@ -88,7 +97,7 @@ int main(int argc, char ** argv) for(int i = 0; i < settings.benchmark.repetitions;) { benchmarker.start(); SPDLOG_DEBUG("Submit execution {}", i); - auto ret = executor.execute(opts.fname, in, out); + auto ret = executor.execute(opts.fname, *in, *out); if(std::get<0>(ret)) { SPDLOG_DEBUG("Finished execution {} out of {}", i, settings.benchmark.repetitions); benchmarker.end(0); @@ -108,7 +117,7 @@ int main(int argc, char ** argv) printf("Data: "); for(int i = 0; i < std::min(100, opts.input_size); ++i) - printf("%d ", ((char*)out.data())[i]); + printf("%d ", ((char*)out->data())[i]); printf("\n"); return 0; diff --git a/rfaas/include/rfaas/rdma_allocator.hpp b/rfaas/include/rfaas/rdma_allocator.hpp new file mode 100644 index 0000000..de1d30b --- /dev/null +++ b/rfaas/include/rfaas/rdma_allocator.hpp @@ -0,0 +1,58 @@ +// +// Created by mou on 4/2/23. +// + +#ifndef __RFAAS_RDMA_ALLOCATOR_HPP__ +#define __RFAAS_RDMA_ALLOCATOR_HPP__ + +#include +#include +#include + +namespace rfaas { + template + class RdmaAllocator { + public: + typedef T value_type; + + inline explicit RdmaAllocator(const executor &executor) noexcept: _executor(executor) {} + + template + constexpr RdmaAllocator(const RdmaAllocator &) noexcept {} + + [[nodiscard]] inline T *allocate(const std::size_t &size, const int &access, int header = 0) { + if (size > std::numeric_limits::max() / sizeof(T)) + throw std::bad_array_new_length(); + + // Maybe we could directly call the memset function here + if (auto buffer = new rdmalib::Buffer(size, header)) { + report(buffer, size); + buffer->register_memory(_executor._state.pd(), access); + return buffer; + } + throw std::bad_alloc(); + } + + inline void deallocate(T *p, std::size_t size) noexcept { + report(p, size, 0); + std::free(p); + } + + private: + const executor &_executor; + + void report(T *p, std::size_t n, bool alloc = true) const { + std::cout << (alloc ? "Alloc: " : "Dealloc: ") << sizeof(T) * n + << " bytes at " << std::hex << std::showbase + << reinterpret_cast(p) << std::dec << '\n'; + } + }; + + template + bool operator==(const RdmaAllocator &, const RdmaAllocator &) { return true; } + + template + bool operator!=(const RdmaAllocator &, const RdmaAllocator &) { return false; } +} + +#endif //__RFAAS_RDMA_ALLOCATOR_HPP__